1 importurllib22 importre3 from pyquery importPyQuery as pq4 from lxml importetree5 importurlparse6 importtime78 allUrls =set()9 allMails =set()10 urlsDownlist =[]1112 classmailCrawler:13 def __in...
1 importurllib22 importre3 from pyquery importPyQuery as pq4 from lxml importetree5 importurlparse6 importtime7
8 allUrls =set()9 allMails =set()10 urlsDownlist =[]11
12 classmailCrawler:13 def __init__(self,mailExpression,start_url,maxcount):14 '''mailExpressoin 邮箱的正则表达式;15 start_url开始邮箱；16 maxcount最大数量'''
17 self.mailpattern =re.compile(mailExpression)18 self.maxcount =maxcount19 self.htmlcount =020 self.UrlsQlist = []#url queue 实现广度优先
21 self.url =start_url22
23
24 defurl_normal(self,url):25 '''url 规范化'''
26 scheme,netloc,path,query = urlparse.urlsplit(url)[:4]27 netloc =netloc.lower()28
29 url.encode("utf-8")30
31 ifpath:32 path = re.sub('/{2,}','/',path)#去除url中的重复/
33 path = re.sub(r'\.$','',path)#去除url中结尾多余的点
34 path = re.sub('/$','&#
最低0.47元/天 解锁文章
2245
被折叠的 条评论
为什么被折叠?
到【灌水乐园】发言
