爬虫

'''高新技术企业认定管理工作网'''

from lxml import etree
import requests, re, time, pymysql


class Spider_GXJSQYRDW_Allday:
    def __init__(self, page_total=14):  # 页面总数14页
        self.page_total = page_total
        self.url = [r'http://www.innocom.gov.cn/gxjsqyrdw/gswj/list.shtml'] + \
                   [r'http://www.innocom.gov.cn/gxjsqyrdw/gswj/list_%d.shtml' % (_ + 2) for _ in range(page_total - 1)]  # 公示公告界面url，共14页
        self.url_homepage = r'http://www.innocom.gov.cn'  # 主页url
        self.save_path = 'C:/Users/admin/Desktop/爬虫附件存放/高新技术企业认定管理工作网-公示公告/%s'
        self.date_today = time.strftime('%Y-%m-%d', time.localtime())
        self.headers = {
            'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.110 Safari/537.36'
        }
        self.website_info = {
            'website_name': '高新技术企业认定管理工作网',
            'block_name': '公示公告'
        }
        self.mysql = {
            'host': 'localhost',
            'user': 'root',
            'pwd': '123456',
            'database': 'test'
        }

    def get_html(self, url):
        '''获取网页源码'''
        r = requests.get(url, headers=self.headers)
        r.encoding = 'utf-8'
        html = etree.HTML(r.text)
        return html

    def get_image(self, url, title):
        '''按顺序下载图片，图片命名按顺序为1-n.jpg(png/gif等)
        :return:
        image_save_path:图片存储路径
        save_successOrNot:图片下载成功与否
        '''
        image_save_path = []
        save_successOrNot = []
        for i, j in enumerate(url[:-1]):
            try:
                r = requests.get(self.url_homepage + j, headers=self.headers)
                image_save_path_temp = self.save_path % (title + '_' + str(i + 1) + j[-4:])
                with open(image_save_path_temp, 'wb') as f:
                    f.write(r.content)
                image_save_path.append(image_save_path_temp)
                save_successOrNot.append('下载成功')
            except:
                image_save_path.append('')
                save_successOrNot.append('下载失败')

        return image_save_path, save_successOrNot

    def get_text_attachment(self, text_url, text_attach_url, text_attach_title):
        '''按顺序下载附件
        :return:
        text_attach_save_path:附件存储路径
        save_successOrNot:附件下载成功与否
        '''
        text_attach_save_path = []
        save_successOrNot = []
        for i, j in enumerate(text_attach_url):
            try:
                text_attach_complete_url = text_url.replace('.shtml', '') + re.search(r"/files(.*?)$", j, re.I).group()
                r = requests.get(text_attach_complete_url, headers=self.headers)
                if text_attach_title[i].count('.pdf') or text_attach_title[i].count('.PDF') or text_attach_title[i].count('.doc'):
                    title = text_attach_title[i]
                else:
                    title = text_attach_title[i] + j[-4:]
                text_attach_save_path_temp = self.save_path % title
                # print(i, j)
                # print(text_attach_complete_url)
                # print(title)
                # print(text_attach_save_path_temp)

                with open(text_attach_save_path_temp, 'wb') as f:
                    f.write(r.content)
                text_attach_save_path.append(text_attach_save_path_temp)
                save_successOrNot.append('下载成功')
            except:
                text_attach_save_path.append('')
                save_successOrNot.append('下载失败')

        return text_attach_save_path, save_successOrNot

    def get_content(self):
        '''
        :return:
        title:文本标题
        date:文本发布日期
        text_url:文本网址
        text:文本正文，二进制存储
        image_exist:当前文本是否存在图片内容
        image_save_successOrNot:图片是否下载成功
        image_save_path:图片存储路径
        text_attach_exist:当前文本是否存在附件
        text_attach_successOrNot:附件是否下载成功
        text_attach_save_path:附件存储路径
        '''
        title = []
        date = []
        text_url = []
        text = []
        image_exist = []
        image_save_successOrNot = []
        image_save_path = []
        text_attach_exist = []
        text_attach_successOrNot = []
        text_attach_save_path = []
        for i in range(self.page_total): # 获取所有网页的所有文本链接
            html = self.get_html(self.url[i])
            title += html.xpath("//div[@class='listbox']//li/a/text()")
            date += html.xpath("//div[@class='listbox']//li/span/text()")
            text_url += [self.url_homepage + _ for _ in html.xpath("//div[@class='listbox']//li/a/@href")] # 获取文本链接
        for j, k in enumerate(text_url): # 遍历每个文本，爬取其内容
            html = self.get_html(k)
            text_temp = html.xpath("//div[@id='content']//text()")
            # text.append(''.join(text_temp).encode('utf-8')) # 获取文本二进制数据
            text.append(''.join(text_temp)) # 获取文本

            # 图片是否存在判断，下载及保存
            image_url = html.xpath("//div[@id='content']//img/@src")
            if image_url == [] or len(image_url) == 1:
                image_exist.append('无图片')
                image_save_successOrNot.append('')
                image_save_path.append('')
            else:
                img_save_path, img_save_successOrNot = self.get_image(image_url, title[j])
                image_exist.append('有%d张图片' % (len(image_url) - 1))
                image_save_successOrNot.append(' & '.join(img_save_successOrNot))
                image_save_path.append(' & '.join(img_save_path))

            # 附件文档是否存在判断，下载及保存
            text_attach_url = html.xpath("//div[@id='content']//a/@href")
            text_attach_title = html.xpath("//div[@id='content']//a/text()")
            if text_attach_url == []:
                text_attach_exist.append('无附件')
                text_attach_successOrNot.append('')
                text_attach_save_path.append('')
            else:
                text_atta_save_path, text_atta_successOrNot = self.get_text_attachment(k, text_attach_url, text_attach_title)
                text_attach_exist.append('有%d个附件' % (len(text_attach_url)))
                text_attach_successOrNot.append(' & '.join(text_atta_successOrNot))
                text_attach_save_path.append(' & '.join(text_atta_save_path))
                # print(text_atta_save_path)
                # print('-' * 50)

        return title, date, text_url, text, text_attach_exist, text_attach_save_path, text_attach_successOrNot, image_exist, image_save_path, image_save_successOrNot

    def to_database(self):
        '''写入数据库'''
        db = pymysql.connect(
            host=self.mysql['host'],
            user=self.mysql['user'],
            password=self.mysql['pwd'],
            database=self.mysql['database'],
            charset='utf8'
        )
        cursor = db.cursor()
        sql = "insert into policy_info(website_name, " \
              "website_block_name, " \
              "text_title, " \
              "text_date," \
              "text_url, " \
              "text_content, " \
              "text_attachment_exist, " \
              "text_attachment_save_path," \
              "get_text_atta_okornot, " \
              "image_exist, " \
              "image_save_path, " \
              "get_image_okornot) " \
              "values (%s, %s, %s, str_to_date(%s, '%%Y-%%m-%%d'), %s, %s, %s, %s, %s, %s, %s, %s);"

        text_info = self.get_content()
        for i in range(len(text_info[0])):
            cursor.execute(sql, (self.website_info['website_name'],
                                 self.website_info['block_name'],
                                 text_info[0][i],
                                 text_info[1][i],
                                 text_info[2][i],
                                 text_info[3][i],
                                 text_info[4][i],
                                 text_info[5][i],
                                 text_info[6][i],
                                 text_info[7][i],
                                 text_info[8][i],
                                 text_info[9][i],
                                 ))

        db.commit()
        cursor.close()
        db.close()

        return text_info

def main_GXJSQYRDW_Allday():
    spider = Spider_GXJSQYRDW_Allday()
    spider.to_database()

if __name__ == '__main__':
    main_GXJSQYRDW_Allday()