介绍
使用urllib的request方法,配合re正则进行静态页面基本数据爬取
代码
import re
from urllib import request
# headers={'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.99 Safari/537.36'}
# req=request.Request('http://www.jianshu.com',headers=headers)
# resp = request.urlopen(req)
# text=resp.read().decode() #获取简书首页数据
# #print(text)
# reg=r'<a class="title"[^>]*>(.*?)</a>' #标题的正则
# regse=r'<p class="abstract".*>\s*(\S*)\s*' #摘要的正则
# pattern=re.compile(reg) #正则匹配
# patternse=re.compile(regse)
# contain=pattern.findall(text,re.M) #寻找所有正则匹配的串
# containse=patternse.findall(text,re.M)
# for (tie,ab) in zip(contain,containse): #联立输出
# print("标题:",tie)
# print("摘要:",ab)
class jianshuhome(object):
def __init__(self):
self.url = 'https://www.jianshu.com'
def get_page(self):
#浏览器标识,基本爬取前缀,不加一般拒绝访问
headers = {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.99 Safari/537.36'}
req = request.Request(self.url, headers=headers)
resp = request.urlopen(req)
data = resp.read().decode()
return data
def parse_data(self, data):
#正则匹配
article = re.findall(r'<a class="title".*?>(.*?)</a>.*?<p class="abstract">(.*?)</p>', data, re.S)
return article
if __name__ == '__main__':
jianshu = jianshuhome()
data = jianshu.get_page()
for i in jianshu.parse_data(data):
print(i[0])
print(i[1])