import requests
from lxml import etree
class Content:
"""
Common base class for all articles/pages
"""
def __init__(self, url, title, body):
self.url = url
self.title = title
self.body = body
def print(self):
"""
Flexible printing function controls output
"""
print('URL: {}'.format(self.url))
print('TITLE: {}'.format(self.title))
print('BODY:\n{}'.format(self.body))
用的是Xpath定位
class Website:
"""
Contains information about website structure
"""
def __init__(self, name, url, x_path,titleTag, bodyTag):
self.name = name
self.url = url
self.x_path = x_path
self.titleTag = titleTag
self.bodyTag = bodyTag
selectedElems = pageObj.xpath(‘string({})’.format(x_path))
这是因为内容页的文字是在一个父标签的各个子标签中都有,所以用‘string()’,注意,string()里的定位不需要引号
class Crawler:
def __init__(self, site):
self.site = site
def getPage(self, url):
user_agent="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.103 Safari/537.36"
headers={"User-Agent":user_agent} #请求头,headers是一个字典类型
html = requests.get(url,headers=headers).content
selector = etree.HTML(html)
return selector
def get_url(self,selector,x_path):
label = selector.xpath(x_path)
labels = []
urls = []
for i in label:
labels.append(i.attrib)
for j in labels:
try:
urls.append(self.site.url+j['href'])
except:
continue
return urls
def safeGet(self, pageObj, x_path):
"""
Utilty function used to get a content string from a Beautiful Soup
object and a selector. Returns an empty string if no object
is found for the given selector
"""
try:
selectedElems = pageObj.xpath('string({})'.format(x_path))
except:
return ''
return selectedElems
def parse(self, url):
"""
Extract content from a given page URL
"""
selector = self.getPage(url)
if selector is not None:
title = self.safeGet(selector, self.site.titleTag)
body = self.safeGet(selector, self.site.bodyTag)
if title != '' and body != '':
content = Content(url, title, body)
content.print()
def crawl(self):
selector = self.getPage(self.site.url)
urls = self.get_url(selector,self.site.x_path)
for url in urls:
self.parse(url)
siteData = [
['kjxy', 'http://kjxy.hbue.edu.cn/', '//*[(@id = "wp_news_w6")]//a', '/html/body/div[2]/div[1]/div[3]/div[2]/div/div[1]/div/div/h1','/html/body/div[2]/div[1]/div[3]/div[2]/div/div[1]/div/div/div[5]/div/div'],
['jrxy', 'http://jrxy.hbue.edu.cn/', '//*[(@id = "wp_news_w3")]//a','//h1', '//*[@id="entry"]/div/div'],
['yjs', 'http://yjs.hbue.edu.cn/', '//*[(@id = "wp_news_w3")]//a', '//h1','//*[@id="mainbody3"]/div[2]/div/div[3]/div/div[1]'],
['mpacc', 'http://mpacc.hbue.edu.cn/', '//*[(@id = "wp_news_w4")]//a', '/html/body/div[1]/div[5]/div[2]/div/div/div[1]','//*[@id="entry"]/div/div']
]
reuter1 = Website(siteData[0][0],siteData[0][1],siteData[0][2],siteData[0][3],siteData[0][4])
reuter2 = Website(siteData[1][0],siteData[1][1],siteData[1][2],siteData[1][3],siteData[1][4])
reuter3 = Website(siteData[2][0],siteData[2][1],siteData[2][2],siteData[2][3],siteData[2][4])
reuter4 = Website(siteData[3][0],siteData[3][1],siteData[3][2],siteData[3][3],siteData[3][4])
crawler = Crawler(reuter1)
crawler.crawl()