爬虫模型作业代码笔记

import requests
from lxml import etree

class Content:
    """
    Common base class for all articles/pages
    """
    def __init__(self, url, title, body):
        self.url = url
        self.title = title
        self.body = body

    def print(self):
        """
        Flexible printing function controls output
        """
        print('URL: {}'.format(self.url))
        print('TITLE: {}'.format(self.title))
        print('BODY:\n{}'.format(self.body))

用的是Xpath定位

class Website:
    """ 
    Contains information about website structure
    """

    def __init__(self, name, url, x_path,titleTag, bodyTag):
        self.name = name
        self.url = url
        self.x_path = x_path
        self.titleTag = titleTag
        self.bodyTag = bodyTag
        

selectedElems = pageObj.xpath(‘string({})’.format(x_path))
这是因为内容页的文字是在一个父标签的各个子标签中都有，所以用‘string()’,注意，string（）里的定位不需要引号

class Crawler:
    def __init__(self, site):
        self.site = site
    def getPage(self, url):
        user_agent="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.103 Safari/537.36"
        headers={"User-Agent":user_agent}  #请求头,headers是一个字典类型
        
        html = requests.get(url,headers=headers).content
        selector = etree.HTML(html)
        return selector
    
    def get_url(self,selector,x_path):
        label = selector.xpath(x_path)
        labels = []
        urls = []
        for i in label:
            labels.append(i.attrib)
            
        for j in labels:
            try:
                urls.append(self.site.url+j['href'])
            except:
                continue
        return urls
    
    def safeGet(self, pageObj, x_path):
        """
        Utilty function used to get a content string from a Beautiful Soup
        object and a selector. Returns an empty string if no object
        is found for the given selector
        """
        
        try:
            selectedElems = pageObj.xpath('string({})'.format(x_path))
        except:
            return ''
        
        return selectedElems

    def parse(self, url):
        """
        Extract content from a given page URL
        """
        selector = self.getPage(url)
        if selector is not None:
            title = self.safeGet(selector, self.site.titleTag)
            body = self.safeGet(selector, self.site.bodyTag)
            if title != '' and body != '':
                content = Content(url, title, body)
                content.print()

    def crawl(self):
        selector = self.getPage(self.site.url)
        urls = self.get_url(selector,self.site.x_path)
        for url in urls:
            self.parse(url)
                        

                

siteData = [
    ['kjxy', 'http://kjxy.hbue.edu.cn/', '//*[(@id = "wp_news_w6")]//a', '/html/body/div[2]/div[1]/div[3]/div[2]/div/div[1]/div/div/h1','/html/body/div[2]/div[1]/div[3]/div[2]/div/div[1]/div/div/div[5]/div/div'],
    ['jrxy', 'http://jrxy.hbue.edu.cn/', '//*[(@id = "wp_news_w3")]//a','//h1', '//*[@id="entry"]/div/div'],
    ['yjs', 'http://yjs.hbue.edu.cn/', '//*[(@id = "wp_news_w3")]//a', '//h1','//*[@id="mainbody3"]/div[2]/div/div[3]/div/div[1]'],
    ['mpacc', 'http://mpacc.hbue.edu.cn/', '//*[(@id = "wp_news_w4")]//a', '/html/body/div[1]/div[5]/div[2]/div/div/div[1]','//*[@id="entry"]/div/div']
]
reuter1 = Website(siteData[0][0],siteData[0][1],siteData[0][2],siteData[0][3],siteData[0][4])

reuter2 = Website(siteData[1][0],siteData[1][1],siteData[1][2],siteData[1][3],siteData[1][4])
reuter3 = Website(siteData[2][0],siteData[2][1],siteData[2][2],siteData[2][3],siteData[2][4])
reuter4 = Website(siteData[3][0],siteData[3][1],siteData[3][2],siteData[3][3],siteData[3][4])


crawler = Crawler(reuter1)
crawler.crawl()