scrapy框架 爬取保存到数据库
import scrapy
class MingxingItem(scrapy.Item):
# define the fields for your item here like:
# name = scrapy.Field()
title = scrapy.Field()
img = scrapy.Field()
设置好需要爬取的程序
import scrapy
from mingxing.items import MingxingItem
class MxSpider(scrapy.Spider):
name = 'mx'
# allowed_domains = ['www.xxx.com']
start_urls = ['http://www.521609.com/tuku/']
# 生成一个通用的url模板
url = 'http://www.521609.com/tuku/index_{}.html'
page_num = 2
def parse(self, response):
print('开始爬取>>>')
li_list = response.xpath('/html/body/div[4]/div[3]/ul/li')
for li in li_list:
title = li.xpath('./a/p/text()')