一 创建爬虫项目
# 进入像创建scrapy项目 的路径 创建scrapy项目
# scrapy startproject scrapy_test
# 进入scrapy项目根路径 创建爬虫
# scrapy genspider 爬虫名 域名
# 运行scrapy项目
# scrapy crawl 爬虫名
二 settings 配置
# 自动带着user-agent
# Crawl responsibly by identifying yourself (and your website) on the user-agent
USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/127.0.0.0 Safari/537.36 Edg/127.0.0.0"
# 只显示响应内容和错误日志(信息)
LOG_LEVEL = 'ERROR'
# 设置为False
# Obey robots.txt rules
ROBOTSTXT_OBEY = False
# 自动带着headers
# Override the default request headers:
#DEFAULT_REQUEST_HEADERS = {
# "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
# "Accept-Language": "en",
#}
ITEM_PIPELINES = {
"lianjiaSpider.pipelines.LianjiaspiderPipeline": 300,
}
三 spider 初始参数分析
# 允许爬取的域名 指定为空则没有限制
allowed_domains = ["xxx.com"]
# 起始url
start_urls = ["xxx"]