1、用基础爬虫爬取糗事百科段子
class QsbkSpider(scrapy.Spider):
name = 'qsbk'
allowed_domains = ['qiushibaike.com']
start_urls = ['https://www.qiushibaike.com/text/page/1/']
base_url = 'https://www.qiushibaike.com'
def parse(self, response):
text_divs = response.xpath("//div[@id='content-left']/div")
for div in text_divs:
author = div.xpath(".//h2/text()").get().strip() #get会解码中文
content = div.xpath(".//span/text()").get().strip()
duanzhi=DuanzhiItem(author=author,content=content)
print(author)
yield duanzhi
next_url = response.xpath("//ul[@class='pagination']/li[last()]/a/@href").get()
if not next_url:
return
else:
yield scrapy.Request(self.base_url+next_url,callback=self.parse)
其中yield scrapy.Request(self.base_url+next_url,callback=self.parse)
会拼接下一页的url继续调用parse函数爬取指定内容
下面是爬取作者内容的一部分 总共有13页
十亿精兵总统领
千户小黄人
小萌新初来乍到
爆笑菌boy
一杯清茶!
恨钱不成山
依丽若水
歌歌哒f
…
2、爬取购房网站的每个省的购房信息
class FangSpider(scrapy.Spider):
name = 'fang'
allowed_domains = ['fang.com']
start_urls = ['https://www.fang.com/SoufunFamily.htm']
def parse(self, response):
trs = response.xpath("//div[@id='c02']//tr")
province = None
for tr in trs:
tds = tr.xpath(".//td[not(@class)]") #寻找没有class的标签
provice_td=tds[0]
provice_text = provice_td.xpath(".//text()").get().strip()
if provice_text!="":
province = provice_text
if province.strip() == '其它