[Python] 纯文本查看 复制代码import requests
import time
import re
from fake_useragent import UserAgent
import multiprocessing as mp
import threading
from queue import Queue
#设置headers
ua = UserAgent(verify_ssl=False)
headers = {
"User-Agent": ua.random,
}
#获取网页文本
def getpage(url):
try:
res = requests.get(url, headers, timeout = 30)
res.encoding = res.apparent_encoding
text = res.text
except:
text = ''
return text
#获取网页状态码
def getcode(url):
try:
res = requests.get(url, headers, timeout = 30)
code = res.status_code
except:
code = 0
return code
# 获取网页中文文本
def getchtext(url,q):
text = getpage(url)
chinese_pattern = '[\u4e00-\u9fa5]+'
chtexts = re.findall(chinese_pattern, text)
chtexts = list(set(chtexts)) #去重
chtexts = " ".join(chtexts)
q.put(chtexts)
#获取网页及关联网页信息
if __name__ == '__main__':
start = time.time()
links = ['http://www.fourd.cn', 'http://www.tfsea.com.cn', 'http://www.csscwshi.com' ,'http://www.marina-zh.com/', 'http://www.gdmoko.cn']
for rawlink in links:
rawtext = getpage(rawlink) # 获取网页信息
#corpchtext = getchtext(rawlink) # 获取网页中文文本信息
link_list = re.findall(r"(?<=href=\").+?(?=\")|(?<=href=\').+?(?=\')", rawtext) # 获取网页内含链接
link_list = list(set(link_list)) # 去除重复项
#获取网页内可访问链接
csurls = []
for link in link_list:
if getcode(link) == 200:
csurl = link
elif getcode('http://' + link) == 200:
csurl = 'http://' + link
elif getcode('https://' + link) == 200:
csurl = 'https://' + link
elif getcode(rawlink + '//' + link) == 200:
csurl = rawlink + link
else:
csurl =""
csurls.append(csurl)
csurls = [x for x in csurls if x != '']
print('关联网址为:\n',csurls)
#多线程模式运行
q = Queue()
threads = []
for i in range(len(csurls)):
t = threading.Thread(target= getchtext, args=(csurls[i], q))
t.start()
threads.append(t)
for thread in threads:
thread.join()
res =[]
for _ in range(len(csurls)):
res.append(q.get())
#汇总所有信息
corptext = " ".join(res) #+ corpchtext
print('公司网页信息汇总:\n',corptext)
end = time.time()
cost = end - start
print('共花费{}s'.format(cost))
#弹窗提醒
from tkinter import messagebox
messagebox.showinfo("提示","信息采集完成!")