python监控demo

 

{
  "urls": [
    {
      "url": "https://www.qianxin.com:443/",
      "headers": {"User-Agent": "Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:66.0) Gecko/20100101 Firefox/66.0",
                     "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
                     "Accept-Language": "zh-CN,zh;q=0.8,zh-TW;q=0.7,zh-HK;q=0.5,en-US;q=0.3,en;q=0.2",
                     "Accept-Encoding": "gzip, deflate", "Connection": "close", "Upgrade-Insecure-Requests": "1"},
      "tagHerfs": [
        "/",
        "/product/index",
        "/product/threat_perception",
        "/product/threat_perception",
        "/product/ngsoc",
        "/product/Safety_management",
        "/product/Log_audit",
        "/product/intelligent_modeling",
        "/product/firewall",
        "/product/firewall",
        "/product/vNGFW",
        "/product/security_isolation",
        "/product/intrustion_detection",
        "/product/intrustion_prevention",
        "/product/terminal_security_mgt",
        "/product/terminal_security_mgt",
        "/product/security_response",
        "/product/nac",
        "/product/secure_desktop",
        "/product/server_system",
        "/product/net_platforms",
        "/product/internet_behavior",
        "/product/internet_behavior",
        "/product/secure_proxy",
        "/product/traffic_management",
        "/product/network_behavior",
        "/product/smart_marketing",
        "/product/cloud_monitoring",
        "/product/cloud_monitoring",
        "/product/mobile_terminal_security",
        "/product/virtualization_security",
        "/product/wireless_intrusion_prevention",
        "/product/code_security",
        "/product/safe_athletics",
        "/product/host_protection",
        "/product/zerotrust",
        "/product/database_audit",
        "/trade/index",
        "/trade/tradegov",
        "/trade/tradeoperator",
        "/trade/tradecentralrate",
        "/trade/tradefinance",
        "/trade/succase",
        "/service/IntAsset_discovery",
        "/threat/threatsafeinstitute",
        "/partner/index",
        "/about/index",
        "javascript:void(0);",
        "http://www.legendsec.com/",
        "https://www.netentsec.com/",
        "http://www.hotjob.cn/wt/qianxin/web/index",
        "/en",
        "/topic_krack",
        "/wannacry",
        "/other/onionwormfix",
        "/sitaw",
        "/trade/tradefinance",
        "/partner/index#notice",
        "/about/newslist",
        "/threat/threatnationnetwork",
        "https://www.butian.net/",
        "/activation/index",
        "/about/purchaseconsulting",
        "/service/MmergencyRes",
        "/product/zerotrust",
        "https://www.butian.net/datacon",
        "/about/index",
        "/partner/index",
        "/about/newslist",
        "/about/purchaseconsulting",
        "/trade/tradegov",
        "/trade/tradeoperator",
        "/trade/tradecentralrate",
        "/trade/tradefinance",
        "https://mybase.qianxin.com/",
        "https://ti.qianxin.com/",
        "https://www.butian.net",
        "https://www.qianxin.com/product/cloud_monitoring",
        "http://www.beian.miit.gov.cn/",
        "http://q.url.cn/CDODuU?_type=wpa&qidian=true",
        "javascript:void(0);"
      ],
      "ignoredHerfs": [
        "/trade/showcase/\\d",
         "/about/shownews/\\d",
        "www.toutiao.com",
        "weibo.com"
      ]
    }
  ]
}

 

 

 

# -*- coding:utf-8 -*-
# main.py
import requests
from pyquery import PyQuery as pq
import  tools
import  os

proxies = {'http': 'proxynj.zte.com.cn:80'}

def MonitorOnePage(webInfo):
    tagHerfs = webInfo['tagHerfs']
    ignoredHerfs = webInfo['ignoredHerfs']
    burp0_url = webInfo['url']
    burp0_headers = webInfo['headers']
    res = requests.get(burp0_url, headers=burp0_headers, proxies=proxies)

    doc = pq(res.content)
    tagAList = doc("a")
    changedTagList =[]
    for tagA in tagAList:
        if "href" in tagA.attrib.keys():
            href = tagA.attrib['href']
            # print (href)
            if href not in tagHerfs and not tools.isHrefInIngoreList(href, ignoredHerfs):
                print ("url:" + burp0_url +  "   has changed, changed info:" +  href)
                changedTagList.append(href)

    if len(changedTagList):
        # 调试 changedTagList
        xx =1


def MonitorOneConfigFile(filename):
    webInfo = tools.loadJson(filename)
    urls = webInfo['urls']
    for url in urls:
        MonitorOnePage(url)

if __name__ == "__main__":
    path = os.getcwd()
    list = os.listdir(path)

    for file in list:
        if file.endswith('json'):
            MonitorOneConfigFile(file)

 

 

 

# -*- coding:utf-8 -*-
# tools.py
import  json
import  re


def isHrefIsIgnore(herf,ignore):
    w = re.findall(ignore, herf)
    if len(w):
        return True
    else :
        return False

def isHrefInIngoreList(href,ignoreList):
    for index in range(len(ignoreList)):
        if isHrefIsIgnore(href, ignoreList[index]):
            return True
    return False


def loadJson(filename):
    with open(filename, 'r') as f:

        return  json.loads(f.read())



if __name__ == "__main__":
    isHrefIsIgnore('/trade/showcase/7?case_type=trade_gov&type=suc_case','/trade/showcse/\d')

    # with open( 'xx.json', 'r') as f:
    #     # temp = yaml.load(f.read())
    #     # print(temp)
    #
    #     xx = json.loads(f.read())
    #
    #
    #     tt=0