python； scrapy框架 爬取保存到数据库

scrapy框架 爬取保存到数据库

import scrapy


class MingxingItem(scrapy.Item):
    # define the fields for your item here like:
    # name = scrapy.Field()

    title = scrapy.Field()
    img = scrapy.Field()

设置好需要爬取的程序

import scrapy
from mingxing.items import MingxingItem


class MxSpider(scrapy.Spider):

    name = 'mx'
    # allowed_domains = ['www.xxx.com']
    start_urls = ['http://www.521609.com/tuku/']

    # 生成一个通用的url模板
    url = 'http://www.521609.com/tuku/index_{}.html'
    page_num = 2

    def parse(self, response):
        print('开始爬取>>>')
        li_list = response.xpath('/html/body/div[4]/div[3]/ul/li')

        for li in li_list:
            title = li.xpath('./a/p/text()')