第7章作业

在安装的窗口输入

pip install scrapy --user -i http://pypi.douban.com/simple --trusted-host pypi.douban.com
 

cd 你的文件夹路径

py -m scrapy startproject TipDMSpider

cd TipDMSpider

在TipDMSpider文件夹里面

更改脚本items.py

import scrapy
class TipdmspiderItem(scrapy.Item):
    title=scrapy.Field()
    text=scrapy.Field()

更改脚本pipelines.py

import pandas as pd
from sqlalchemy import create_engine
class TipdmspiderPipeline(object):
    def _init_(self):
        self.engine=create_engine('mysql+pymysql://root:335210@127.0.0.1:3306/tipdm')
    def process_item(self,item,spider):
        data=pd.DataFrame(dict(item))
        data.to_sql('tipdm_data',self.engine,if_exists='append',index=False)
        data.to_csv('TipDM_data.csv',mode='a+',index=False,sep='|',header=False)
        

py -m scrapy genspider tipdm www.tipdm.com

settings.py添加

ROBOTSTXT_OBEY=False
DOWNLOAD_DELAY=5
ITEM_PIPELINES={
    'TipDMSpider.pipelines.TipdmspiderPipeline':300,
}
HTTPCACHE_ENABLED=True
HTTPCACHE_DIR='D:/class/class/爬虫/7/TipDMSpider'

更改tipdm.py

import scrapy
from scrapy.http import Request
from TipDMSpider.items import TipdmspiderItem

class TipdmSpider(scrapy.Spider):
    name = 'tipdm'
    allowed_domains = ['www.tipdm.com']
    start_urls = ['http://www.tipdm.com/']

    def parse(self, response):
        last_page_num=response.xpath("//div[@class='fpage']/div/a[last()]/text()").extract()
        append_urls=['http://www.tipdm.com/tipdm/tddt/index_%d.html'%i\
                     for i in range(2,int(last_page_num[0])+1)]
        append_urls.append('http://www.tipdm.com/tipdm/tddt')
        headers = {
            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.4844.51 Safari/537.36',
        }
        for url in append_urls:
            yield Request(url,callback=self.parse_url,dont_filter=True,headers=headers)
        pass
    def parse_url(self,response):
        urls=response.xpath("//div[@class='item clearfix']/div[1]/h1/a/@herf").extract()
        headers = {
            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.4844.51 Safari/537.36',
        }
        for page_url in urls:
            text_url="http://www.tipdm.com"+page_url
            yield Request(text_url, callback=self.parse_text, dont_filter=True, headers=headers)
        pass
    def parse_text(self,response):
        item=TipdmspiderItem()
        item['title']=response.xpath("//div[@class'artTitle']/h1/text()").extract()
        text=response.xpath("//div[@class'artCon']//p/text()").extract()
        texts=" "
        for strings in text:
            texts=texts+strings+"\n"

        yield item

py -m scrapy crawl tipdm

结果如下图 所示

 提示：如果有安装包没有安装使用

pip install 包的名字 --user -i http://pypi.douban.com/simple --trusted-host pypi.douban.com

安装速度会很快