在安装的窗口输入
pip install scrapy --user -i http://pypi.douban.com/simple --trusted-host pypi.douban.com
cd 你的文件夹路径
py -m scrapy startproject TipDMSpider
cd TipDMSpider
在TipDMSpider文件夹里面
更改脚本items.py
import scrapy
class TipdmspiderItem(scrapy.Item):
title=scrapy.Field()
text=scrapy.Field()
更改脚本pipelines.py
import pandas as pd
from sqlalchemy import create_engine
class TipdmspiderPipeline(object):
def _init_(self):
self.engine=create_engine('mysql+pymysql://root:335210@127.0.0.1:3306/tipdm')
def process_item(self,item,spider):
data=pd.DataFrame(dict(item))
data.to_sql('tipdm_data',self.engine,if_exists='append',index=False)
data.to_csv('TipDM_data.csv',mode='a+',index=False,sep='|',header=False)
py -m scrapy genspider tipdm www.tipdm.com
settings.py添加
ROBOTSTXT_OBEY=False
DOWNLOAD_DELAY=5
ITEM_PIPELINES={
'TipDMSpider.pipelines.TipdmspiderPipeline':300,
}
HTTPCACHE_ENABLED=True
HTTPCACHE_DIR='D:/class/class/爬虫/7/TipDMSpider'
更改tipdm.py
import scrapy
from scrapy.http import Request
from TipDMSpider.items import TipdmspiderItem
class TipdmSpider(scrapy.Spider):
name = 'tipdm'
allowed_domains = ['www.tipdm.com']
start_urls = ['http://www.tipdm.com/']
def parse(self, response):
last_page_num=response.xpath("//div[@class='fpage']/div/a[last()]/text()").extract()
append_urls=['http://www.tipdm.com/tipdm/tddt/index_%d.html'%i\
for i in range(2,int(last_page_num[0])+1)]
append_urls.append('http://www.tipdm.com/tipdm/tddt')
headers = {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.4844.51 Safari/537.36',
}
for url in append_urls:
yield Request(url,callback=self.parse_url,dont_filter=True,headers=headers)
pass
def parse_url(self,response):
urls=response.xpath("//div[@class='item clearfix']/div[1]/h1/a/@herf").extract()
headers = {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.4844.51 Safari/537.36',
}
for page_url in urls:
text_url="http://www.tipdm.com"+page_url
yield Request(text_url, callback=self.parse_text, dont_filter=True, headers=headers)
pass
def parse_text(self,response):
item=TipdmspiderItem()
item['title']=response.xpath("//div[@class'artTitle']/h1/text()").extract()
text=response.xpath("//div[@class'artCon']//p/text()").extract()
texts=" "
for strings in text:
texts=texts+strings+"\n"
yield item
py -m scrapy crawl tipdm
结果如下图 所示
提示:如果有安装包没有安装使用
pip install 包的名字 --user -i http://pypi.douban.com/simple --trusted-host pypi.douban.com
安装速度会很快