第一步:书写jobbole.py
# -*- coding: utf-8 -*-
import scrapy
class JobboleSpider(scrapy.Spider):
name = 'jobbole'
allowed_domains = ["www.hexun.com"]
start_urls = ['http://www.hexun.com']
def parse(self, response):
# 这里选择的网站为http开头的,并且该网站能访问
re_selector = response.xpath("//div[@class='czg-title']/text()").extract()[0]
print (re_selector)
print ('123')
pass
第二步:书写main.py(进行启动用)
from scrapy.cmdline import execute
import sys
import os
sys.path.append(os.path.dirname(os.path.abspath(__file__)))
execute(["scrapy", "crawl", "jobbole"])
第三步:将settings.py的文件进行修改
ROBOTSTXT_OBEY = False