aiohttp保存MySQL_基于asyncio、aiohttp、xpath的异步爬虫

今天介绍一下基于asyncio和aiohttp的异步爬虫的编写，解析html用的是xpath。

该爬虫实现了以下功能:

1.读取csv文件中的爬取规则，根据规则爬取数据；代码中添加了对3个网站的不同提取规则，如有需要，还可以继续添加；

2.将爬取到的数据保存到mysql数据库中。

通过输入问题，该爬虫能爬取关于健康方面的数据。

具体代码如下:

# coding:utf-8

"""

async-apiser xpath

"""

from lxml import etree

import csv

import re

import os

import asyncio

import aiohttp

import aiomysql

from datetime import datetime

from config import Config

class HealthSpider(object):

def __init__(self, user_id, keyword, url, hrule, drule, count, trule):

self.user_id = user_id

self.keyword = keyword

self.url = url

self.hrule = hrule

self.drule = drule

self.count = count

self.trule = trule

self.headers = ''

self.urls_done = []

self.urls_will = []

self.spider_data = {}

@staticmethod

def handle_flag(str):

"""

去除字符串中的style样式标签

:param html:

:return:

"""

pattern = re.compile(r' style=".*?;"', re.S)

return pattern.sub('', str)

async def get_html(self, url, session):

"""

根据url，返回html

:param url:

:return:

"""

try:

async with session.get(url, headers=self.headers, timeout=5) as resp:

if resp.status in [200, 201]:

data = await resp.text()

return data

except Exception as e:

raise Exception("数据搜索错误")

def get_url(self, resp):

"""

根据html获取每条数据的url

:param resp:

:return:

"""

# 保存爬取的数据

root = etree.HTML(str(resp))

items = root.xpath(self.hrule)

# html结构不同，组织url的方式也不同

if 5 == self.count:

self.urls_will = ['https://dxy.com' + i for i in items[:5]]

elif 3 == self.count:

self.urls_will = [i for i in items[:3]]

elif 2 == self.count:

self.urls_will = [i for i in items[:2]]

async def get_data(self, url, session, pool):

"""

根据url获取具体数据

:return:

"""

# 根据url解析出htm

html = await self.get_html(url, session)

# 保存爬取的数据

root = etree.HTML(str(html))

html_data = ''

try:

title = root.xpath(self.trule)

title = ''.join(title)

except Exception as e:

title = ''

try:

data = root.xpath(self.drule)

if data:

# html结构不同，获取数据的方式也不同

if 3 == self.count:

html_data = ''.join(map(etree.tounicode, data))

# 去除结果中的style标签

html_data = HealthSpider.handle_flag(html_data)

else:

html_data = etree.tounicode(data[0])

html_data = HealthSpider.handle_flag(html_data)

except Exception as e:

html_data = []

self.urls_done.append(url)

# 数据入库,保存：用户id, 关键词, 日期, 主url, 子url, html数据

if html_data:

self.spider_data["data"].append({"title": title, "html_data": html_data})

spide_date = datetime.now()

data = (self.user_id, self.keyword, spide_date, self.url, url, title, html_data)

stmt = "INSERT INTO spider_data (user_id, keyword, spide_date, main_url, sub_url, title, html_data) " \

"VALUES (%s, %s, %s, %s, %s, %s, %s)"

try:

async with pool.acquire() as conn:

async with conn.cursor() as cur:

await cur.execute(stmt, data)

except Exception as e:

pass

async def start_spider(self, pool):

"""

开始爬取数据

:return:

"""

async with aiohttp.ClientSession() as session:

self.spider_data["user_id"] = self.user_id

self.spider_data["keyword"] = self.keyword

self.spider_data["data"] = []

while True:

# 待爬取url队列为空或者已经爬取3条数据,则停止爬取

if (len(self.urls_will) == 0) or len(self.spider_data["data"]) == self.count:

break

# 获取待爬url

url = self.urls_will.pop()

# 开始爬取数据

if url not in self.urls_done:

await self.get_data(url, session, pool)

return self.spider_data

async def main(self, loop):

# 请求头

self.headers = {'Accept': 'text/html, application/xhtml+xml, application/xml;q=0.9,*/*;q=0.8',

'Accept-Encoding': 'gzip, deflate',

'Accept-Language': 'zh-Hans-CN, zh-Hans; q=0.5',

'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 '

'(KHTML, like Gecko) Chrome/52.0.2743.116 Safari/537.36 Edge/15.15063'

}

# 连接mysql数据库

pool = await aiomysql.create_pool(host=Config.DB_HOST, port=Config.DB_PORT,

user=Config.DB_USER, password=Config.DB_PASSWORD,

db=Config.DB_NAME, loop=loop, charset="utf8", autocommit=True)

async with aiohttp.ClientSession() as session:

# 首次获取html

html = await self.get_html(self.url, session)

# 获取url

self.get_url(html)

data = await self.start_spider(pool)

return data

# asyncio.ensure_future(self.start_spider(pool))

def get_rules(keyword):

"""

获取csv中的xpath规则

:return:

"""

csv_dict = []

path = os.path.join(os.path.dirname(__file__), 'rules.csv')

with open(path, 'rU') as f:

reader = csv.DictReader(f)

for line in reader:

url = line['url'].format(keyword)

hrule = line['hrule']

drule = line['drule']

count = int(line['count'])

title = line['trule']

csv_dict.append({"url": url, "hrule": hrule, "drule": drule, "count": count, "trule": title})

return csv_dict

def start_spider(keyword):

"""

爬取数据

:param user_id:

:param keyword:

:return:

"""

try:

data_list = get_rules(keyword)

except Exception as e:

raise Exception("搜索规则获取失败")

spider_data = []

tasks = []

loop = asyncio.get_event_loop()

for i in data_list:

spider = HealthSpider(1, keyword, i['url'], i['hrule'], i['drule'], i['count'], i['trule'])

# 任务列表

tasks.append(asyncio.ensure_future(spider.main(loop)))

# 添加到loop

loop.run_until_complete(asyncio.wait(tasks))

try:

for task in tasks:

for i in range(len(task.result()["data"])):

spider_data.append(task.result()["data"][i])

except Exception as e:

pass

# 延时以等待底层打开的连接关闭

loop.run_until_complete(asyncio.sleep(0.250))

loop.close()

return spider_data

if __name__ == '__main__':

# 爬取感冒了怎么办相关内容

start_spider("感冒了怎么办")

下面讲一下代码中某些方法的作用:

1.handle_flag()方法用于去掉html字符串中的style样式标签，保留html中的其他标签，便于前端的展示；

2.get_data()方法用于爬取具体数据，并使用aiomysql将爬取道德数据保存到数据库；

数据库的配置文件config.py:

# coding=utf-8

class Config(object):

DB_ENGINE = 'mysql'

DB_HOST = '127.0.0.1'

DB_PORT = 3306

DB_USER = 'root'

DB_PASSWORD = 'wyzane'

DB_NAME = 'db_tornado'

DB_OPTIONS = {

'init_command': "SET sql_mode='STRICT_TRANS_TABLES'",

'charset': 'utf8mb4',

}

3.get_rules()方法用于从rules.csv文件中读取爬取的规则。因为这里同时爬取了3个不同的网站，由于每个网站解析html的xpath规则不同，并且每个网站提取的数据条数不同，所以把这些规则写到了rules.csv文件(就是一个excel文件)中。先读取规则，再爬取数据。

以上就是基于asyncio的异步爬虫的代码，如有错误，欢迎交流指正！