aiohttp mysql_使用aiohttp实现爬虫

importasyncioimportreimportaiohttpimportaiomysqlfrom pyquery importPyQuery

stopping=False

start_url= "http://www.jobbole.com/"waitting_urls=[]

seen_urls=set()

sem= asyncio.Semaphore(3)

asyncdeffetch(url, session):

async with sem:

await asyncio.sleep(1)try:

async with session.get(url) as response:print("Status:", response.status)print("Content-type:", response.headers[‘content-type‘])if response.status in [200, 201]:

content=await response.text()returncontentexceptException as e:print(e)defextract_urls(html):

pq=PyQuery(html)for link in pq.items("a"):

url= link.attr("href")if url and url.startswith("/caijing") and url not inseen_urls:

waitting_urls.append("http://www.jobbole.com{}".format(url))

asyncdefinit_urls(url, session):

html=await fetch(url, session)

seen_urls.add(url)

extract_urls(html)

asyncdefarticle_handler(url, session, pool):#获取文章详情并解析入库

html =await fetch(url, session)

seen_urls.add(url)

extract_urls(html)

pq=PyQuery(html)

title= pq("title").text()

async with pool.acquire() as conn:

async with conn.cursor() as cur:

insert_sql= "insert into artest(title) values (‘{}‘)".format(title)

await cur.execute(insert_sql)

asyncdefconsumer(pool):

async with aiohttp.ClientSession() as session:while notstopping:if len(waitting_urls) ==0:

await asyncio.sleep(0.5)continueurl=waitting_urls.pop()print("start get url: {}".format(url))if re.match("http://.*?jobbole.com/.*\.html", url):if url not inseen_urls:

asyncio.ensure_future(article_handler(url, session, pool))

await asyncio.sleep(0.5)else:if url not inseen_urls:

asyncio.ensure_future(init_urls(url, session))

asyncdefmain(loop):

pool= await aiomysql.create_pool(host=‘‘, port=,

user=‘‘, password=‘‘,

db=‘‘, loop=loop,

charset="utf8", autocommit=True)

async with aiohttp.ClientSession() as session:

html=await fetch(start_url, session)

seen_urls.add(start_url)

extract_urls(html)

asyncio.ensure_future(consumer(pool))if __name__ == "__main__":

loop=asyncio.get_event_loop()

asyncio.ensure_future(main(loop))

loop.run_forever()