python批量测试网站加载时间_【笔记】Python 批量检测网站存活

2018年5月份写了一版，但是很慢，期间也改过很多版，但还是很慢。

2019年9月20日改了一下，先检测端口是否存活，存活了再去探测web：

使用方法：

url.py C:\Python36\test\url.txt

程序 链接文本路径（一行一个）

import requests

import queue

import re

import sys

import time

import threading

import telnetlib

from urllib.parse import urlparse

requests.packages.urllib3.disable_warnings()

class Waittask(threading.Thread):

def run(self):

self.telnet = telnetlib.Telnet()

while True:

if asset.qsize() < 100:

for i in range(500):

host = urlparse(waittask.get())

scheme = host.scheme

netloc = host.netloc

try:

if ':' in netloc:

Host = netloc.split(':')

self.telnet.open(Host[0],Host[1],timeout = 3)

asset.put('{}://{}:{}'.format(scheme,Host[0],Host[1]))

else:

self.telnet.open(netloc,80,timeout = 3)

asset.put('{}://{}'.format(scheme,netloc))

except Exception as e:

pass

time.sleep(1)

class Asset(threading.Thread):

def run(self):

while True:

while not asset.empty():

url = asset.get()

try:

response = requests.get(url, verify=False, allow_redirects=False,timeout = 15)

try:

title = re.findall('

(.*?)',response.text.lower())[0]

except Exception as e:

title = '获取标题失败'

print(response.status_code,response.url,title)

with open('survival.txt','a',encoding='gb18030') as cent:

cent.write('{}|{}|{}\n'.format(response.status_code,response.url,title))

except Exception as e:

pass

time.sleep(1)

class Check(threading.Thread):

def run(self):

while True:

time.sleep(10)

print('进度：{:.2%} 当前还有{}个端口开放检测，{}个WEB探测任务未完成！'.format(1-int(waittask.qsize())/sums,waittask.qsize(),asset.qsize()))

if __name__ == '__main__':

file = sys.argv[1]

waittask = queue.Queue()

asset = queue.Queue()

with open(file) as content:

for cent in content:

waittask.put(cent.strip())

sums = int(waittask.qsize())

for i in range(100):

Waittask_ = Waittask()

Waittask_.start()

for i in range(100):

Asset_ = Asset()

Asset_.start()

check = Check()

check.start()