两个爬虫记录
第一个
from urllib
import request
import re
import datetime
# import gzip
# from io import BytesIO
class Spider():
# now_time = datetime.datetime.now
url = 'http://www.shfe.com.cn/news/notice/'
# 链接
# root_pattern = '<div class="p4 lawbox">([\s\S]*?)</div>'
root_pattern = '<li>([\s\S]*?)</li>'
# 根目录
time_patten = '<span>([\s\S]*?)</span>'
# 时间
title_pattern = '">([\w\W]*?)</a>'
# title
def __fetch_content(self):
# 1
r = request.urlopen(Spider.url)
htmls = r.read()
htmls = str(htmls,encoding='utf-8')
# print(htmls)
return htmls
def __analysis(self,htmls):
root_html = re.findall(Spider.root_pattern,htmls)
anchors = []
for html in root_html:
title = re.findall(Spider.title_pattern,html)
time = re.findall(Spider.time_patten,html)
anchor = {'time':time,'title':title}
anchors.append(anchor)
# print(root_html[0])
#print(anchors)
# a = 1
return anchors
def __refine(self,anchors):
l = lambda anchor:{
'time':anchor['time'][0].strip(),
'title':anchor['title'][0]
}
return map(l,anchors)
# 数据精炼
# strip()方法用来去除字符串中的 前后的空格 升序排列
# reverse=True 变成降序配列
def __sort(self,anchors):
anchors = sorted(anchors,key=self.__sort_seed,reverse=True)
return anchors
# 业务处理
# sorted()方法排序
# key用来指定用来比较大小的元素
def __sort_seed(self,anchor):
return anchor['time']
# 时间的比较运算
def __show(self,anchors):
for anchor in anchors:
print(anchor['title'] + '------' + anchor['time'])# 展示sort
def go(self):
# 2
# 入口方法
htmls = self.__fetch_content()
# 调用__fetch_content函数
anchors = self.__analysis(htmls)
anchors = list(self.__refine(anchors))
anchors = self.__sort(anchors)
# print(anchors)
self.__show(anchors)
spider = Spider()
spider.go()
第二个
from urllib
import request
import re
import datetime
# import gzip
# from io import BytesIO
class Spider():
url = 'http://www.cffex.com.cn/jysgg/'
# root_pattern_cffex = '<ul class="clearFloat">([\s\S]*?)</ul>'
root_pattern_cffex = 'class="time comparetime">([\w\W]*)</a>'
time_pattern_cffex = 'class="time comparetime">([\w\W]*?)</a>'
title_pattern_cffex = 'title="([\w\W]*?)" >'
def __fetch_content_cffex(self):
r = request.urlopen(Spider.url)
htmls = r.read()
htmls = str(htmls, encoding='utf-8')
# print(htmls)
return htmls
# a = 1
def __analysis_cffex(self, htmls):
# 分析
root_ceffex = re.findall(Spider.root_pattern_cffex, htmls)
anchors = []
for html in root_ceffex:
time = re.findall(Spider.time_pattern_cffex, html)
title = re.findall(Spider.title_pattern_cffex, html)
anchor = {'title': title,'time': time }
anchors.append(anchor)
tit = anchor.get('title')
tim = anchor.get('time')
i = 0
while i <len(tit):
print(tit[i], end = '\t')
print(tim[i])
i += 1
# print(anchor)
return anchors
# print(htmls)
# a = 1
def __refine(self, anchors):
pass
def go(self):
htmls = self.__fetch_content_cffex()
# self.__analysis_cffex(htmls)
anchors = self.__analysis_cffex(htmls)
self.__refine(anchors)
Spider = Spider()
Spider.go()
po完了 凑合看:)