scrapy爬虫之贝壳房产信息
今天分享的代码是scrapy框架爬取贝壳房产的信息,保存到mysql中,然后通过pyecharts进行数据分析得到html,最终通过整个html得到完整的数据分析网页。还请各位看官有错指错,大家一起学习进步~话不多说,直接上代码!
一、scrapy框架代码:
1、item.py
import scrapy
class BeikeItem(scrapy.Item):
# define the fields for your item here like:
# name = scrapy.Field()
biaoti = scrapy.Field()
didian = scrapy.Field()
huxing = scrapy.Field()
jiage = scrapy.Field()
danjia = scrapy.Field()
biaoqian = scrapy.Field()
mianji = scrapy.Field()
2、middlewares.py
from scrapy import signals
# useful for handling different item types with a single interface
from itemadapter import is_item, ItemAdapter
class BeikeSpiderMiddleware:
# Not all methods need to be defined. If a method is not defined,
# scrapy acts as if the spider middleware does not modify the
# passed objects.
@classmethod
def from_crawler(cls, crawler):
# This method is used by Scrapy to create your spiders.
s = cls()
crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)
return s
def process_spider_input(self, response, spider):
# Called for each response that goes through the spider
# middleware and into the spider.
# Should return None or raise an exception.
return None
def process_spider_output(self, response, result, spider):
# Called with the results returned from the Spider, after
# it has processed the response.
# Must return an iterable of Request, or item objects.
for i in result:
yield i
def process_spider_exception(self, response, exception, spider):
# Called when a spider or process_spider_input() method
# (from other spider middleware) raises an exception.
# Should return either None or an iterable of Request or item objects.
pass
def process_start_requests(self, start_requests, spider):
# Called with the start requests of the spider, and works
# similarly to the process_spider_output() method, except
# that it doesn’t have a response associated.
# Must return only requests (not items).
for r in start_requests:
yield r
def spider_opened(self, spider):
spider.logger.info('Spider opened: %s' % spider.name)
class BeikeDownloaderMiddleware:
# Not all methods need to be defined. If a method is not defined,
# scrapy acts as if the downloader middleware does not modify the
# passed objects.
@classmethod
def from_crawler(cls, crawler):
# This method is used by Scrapy to create your spiders.
s = cls()
crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)
return s
def process_request(self, request, spider):
# Called for each request that goes through the downloader
# middleware.
# Must either:
# - return None: continue processing this request
# - or return a Response object
# - or return a Request object
# - or raise IgnoreRequest: process_exception() methods of
# installed downloader middleware will be called
return None
def process_response(self, request, response, spider):
# Called with the response returned from the downloader.
# Must either;
# - return a Response object
# - return a Request object
# - or raise IgnoreRequest
return response
def process_exception(self, request, exception, spider):
# Called when a download handler or a process_request()
# (from other downloader middleware) raises an exception.
# Must either:
# - return None: continue processing this exception
# - return a Response object: stops process_exception() chain
# - return a Request object: stops process_exception() chain
pass
def spider_opened(self, spider):
spider.logger.info('Spider opened: %s' % spider.name)
3、pipelines.py
# Define your item pipelines here
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: https://docs.scrapy.org/en/latest/topics/item-pipeline.html
# useful for handling different item types with a single interface
from itemadapter import ItemAdapter
import pymysql
class BeikePipeline:
def __init__(self):
self.connect = pymysql.connect(host="localhost", user="root", passwd="1234", db="beike")
self.cursor = self.connect.cursor()
print("数据库连接成功")
def process_item(self, item, spider):
print("开始保存数据")
insql = "insert into beike_beijing(biaoti,didian,huxing,jiage,danjia,biaoqian,mianji) values (%s,%s,%s,%s,%s,%s,%s)"
self.cursor.execute(insql, (
item['biaoti'], item['didian'], item['huxing'], item['jiage'], item['danjia'],
item['biaoqian'], item['mianji']))
self.connect.commit()
print("保存数据成功")
return item
def parse_close(self):
self.connect.close()
self.cursor.close()
4、settings.py
# Scrapy settings for beike project
#
# For simplicity, this file contains only settings considered important or
# commonly used. You can find more settings consulting the documentation:
#
# https://docs.scrapy.org/en/latest/topics/settings.html
# https://docs.scrapy.org/en/latest/topics/downloader-middleware.html
# https://docs.scrapy.org/en/latest/topics/spider-middleware.html
BOT_NAME = 'beike'
SPIDER_MODULES = ['beike.spiders']
NEWSPIDER_MODULE = 'beike.spiders'
# Crawl responsibly by identifying yourself (and your website) on the user-agent
# USER_AGENT = 'beike (+http://www.yourdomain.com)'
# Obey robots.txt rules
ROBOTSTXT_OBEY = False
# Configure maximum concurrent requests performed by Scrapy (default: 16)
# CONCURRENT_REQUESTS = 32
# Configure a delay for requests for the same website (default: 0)
# See https://docs.scrapy.org/en/latest/topics/settings.html#download-delay
# See also autothrottle settings and docs
# DOWNLOAD_DELAY = 3
# The download delay setting will honor only one of:
# CONCURRENT_REQUESTS_PER_DOMAIN = 16
# CONCURRENT_REQUESTS_PER_IP = 16
# Disable cookies (enabled by default)
COOKIES_ENABLED = True
# Disable Telnet Console (enabled by default)
# TELNETCONSOLE_ENABLED = False
# Override the default request headers:
DEFAULT_REQUEST_HEADERS = {
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
'Accept-Language': 'en',
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36',
'Cookie': 'digv_extends=%7B%22utmTrackId%22%3A%2280418643%22%7D; lianjia_uuid=3d726c57-6d3f-4f6c-95a2-8b7abc9faeac; select_city=110000; lianjia_ssid=4473e3e6-43e7-4181-bb6a-ebb23ef4ec07; sajssdk_2015_cross_new_user=1; sensorsdata2015jssdkcross=%7B%22distinct_id%22%3A%221769ce5aab93e7-0c28ca07c23265-59442e11-1327104-1769ce5aaba695%22%2C%22%24device_id%22%3A%221769ce5aab93e7-0c28ca07c23265-59442e11-1327104-1769ce5aaba695%22%2C%22props%22%3A%7B%22%24latest_traffic_source_type%22%3A%22%E4%BB%98%E8%B4%B9%E5%B9%BF%E5%91%8A%E6%B5%81%E9%87%8F%22%2C%22%24latest_referrer%22%3A%22https%3A%2F%2Fwww.baidu.com%2Fother.php%22%2C%22%24latest_referrer_host%22%3A%22www.baidu.com%22%2C%22%24latest_search_keyword%22%3A%22%E8%B4%9D%E5%A3%B3%22%2C%22%24latest_utm_source%22%3A%22baidu%22%2C%22%24latest_utm_medium%22%3A%22pinzhuan%22%2C%22%24latest_utm_campaign%22%3A%22wymoren%22%2C%22%24latest_utm_content%22%3A%22biaotimiaoshu%22%2C%22%24latest_utm_term%22%3A%22biaoti%22%7D%7D; crosSdkDT2019DeviceId=sgaxf7--ohhq7q-mq2s3hm3qk16atd-otg4hxhsr; _ga=GA1.2.999642397.1608950071; _gid=GA1.2.1417890696.1608950071; __xsptplusUT_788=1; __xsptplus788=788.1.1608950072.1608950072.1%234%7C%7C%7C%7C%7C%23%23duwbmR1LtYCy9OIqePHhHWS1htLXHyiz%23; Hm_lvt_9152f8221cb6243a53c83b956842be8a=1608950065,1608950073,1608950215; Hm_lpvt_9152f8221cb6243a53c83b956842be8a=1608950261; srcid=eyJ0Ijoie1wiZGF0YVwiOlwiM2M1NmJhYjliNzhmYzhhYzYzYWUyZGVjOWZmZWJjMjQwYzJhZmFlYmRjZTk4YWU2M2E3MDU4MjY3MDFlNDc5MThlNzkwMDI1NWM4NzNkYTA5YmQyZjBkZDFjZGIxZDg1YmJkMDlmODlmYzFkZGQxOTNiNGI3ZGU5MTU5ZmZlYWVlNWJlMjIzNTFkNzk2NDJkOTI4ZDYzYWEzNjkwYTVlNGU3MDRhMDcxYzQ5NDhmN2RiMzdjMGZiZGExZGY3NzdlZjYyMWZkOGMwMTAzMGNlZmUxNWZmYzAyMjlkODA0MTczZjE1MGRmOTFiYjZjZTgzNDEyY2JlOThjNDMwYzI1YjU2NGI2M2Q4ZTUxZjA5ZmM5MTgyMGVjZWY2OTA2ZDhkN2JiYWYxMzFkZDkxZjU3YjUxZWZhNTZjM2EyNzczMGI4ODgxNGFhNGViNjA5YjlhMjMxYmI0OWZiNzEyNzBhNFwiLFwia2V5X2lkXCI6XCIxXCIsXCJzaWduXCI6XCJjNDBjMDg1ZVwifSIsInIiOiJodHRwczovL2JqLmtlLmNvbS9lcnNob3VmYW5nL3BnMi8iLCJvcyI6IndlYiIsInYiOiIwLjEifQ==; login_ucid=2000000074667028; lianjia_token=2.0015e8780f68fe41f70445513e50d1f7b5; lianjia_token_secure=2.0015e8780f68fe41f70445513e50d1f7b5; security_ticket=WyjQtDuz1ImoP8myKzaHDGUewY7FuWIViEWxA+VfVYPS1kh3NigeIWicj7EQoTgPFJUTK6nPMHlbU+pvTlI4XRKfiyiRoeyEjIqFkcidofJneE75XwFlyXW1/eb85/AktQwvEFK2zqJHTb5owtGQiVxFGh2l/UFVDVJMjHsN4Ec='
}
# Enable or disable spider middlewares
# See https://docs.scrapy.org/en/latest/topics/spider-middleware.html
# SPIDER_MIDDLEWARES = {
# 'beike.middlewares.BeikeSpiderMiddleware': 543,
# }
# Enable or disable downloader middlewares
# See https://docs.scrapy.org/en/latest/topics/downloader-middleware.html
# DOWNLOADER_MIDDLEWARES = {
# 'beike.middlewares.BeikeDownloaderMiddleware': 543,
# }
# Enable or disable extensions
# See https://docs.scrapy.org/en/latest/topics/extensions.html
# EXTENSIONS = {
# 'scrapy.extensions.telnet.TelnetConsole': None,
# }
# Configure item pipelines
# See https://docs.scrapy.org/en/latest/topics/item-pipeline.html
ITEM_PIPELINES = {
'beike.pipelines.BeikePipeline': 300,
}
# Enable and configure the AutoThrottle extension (disabled by default)
# See https://docs.scrapy.org/en/latest/topics/autothrottle.html
# AUTOTHROTTLE_ENABLED = True
# The initial download delay
AUTOTHROTTLE_START_DELAY = 5
# The maximum download delay to be set in case of high latencies
AUTOTHROTTLE_MAX_DELAY = 60
# The average number of requests Scrapy should be sending in parallel to
# each remote server
AUTOTHROTTLE_TARGET_CONCURRENCY = 1.0
# Enable showing throttling stats for every response received:
AUTOTHROTTLE_DEBUG = False
# Enable and configure HTTP caching (disabled by default)
# See https://docs.scrapy.org/en/latest/topics/downloader-middleware.html#httpcache-middleware-settings
# HTTPCACHE_ENABLED = True
# HTTPCACHE_EXPIRATION_SECS = 0
# HTTPCACHE_DIR = 'httpcache'
# HTTPCACHE_IGNORE_HTTP_CODES = []
# HTTPCACHE_STORAGE = 'scrapy.extensions.httpcache.FilesystemCacheStorage'
5、beikebeijing.py
import scrapy
from beike.items import BeikeItem
import time
class BeikebeijingSpider(scrapy.Spider):
n_id = 2
name = 'beikebeijing'
allowed_domains = ['ke.com']
start_urls = ['https://bj.ke.com/ershoufang/pg1/']
def parse(self, response):
item = BeikeItem()
for bk in response.xpath("//div[@data-component='list']/ul/li"):
biaoti = bk.xpath(".//div[@class='info clear']/div[@class='title']/a/text()").extract()
# time.sleep(1)
if len(biaoti) > 0:
item['biaoti'] = biaoti[0]
else:
item['biaoti'] = ""
# print(biaoti)
didian = bk.xpath(".//div[@class='positionInfo']/a/text()").extract()
time.sleep(1)
if len(didian) > 0:
item['didian'] = didian[0]
else:
item['didian'] = ""
# print(didian)
huxing = bk.xpath(".//div[@class='houseInfo']/text()").extract()
# time.sleep(1)
if len(huxing) > 0:
item['huxing'] = huxing[0]
else:
item['huxing'] = ""
# print(huxing)
jiage = bk.xpath(".//div[@class='totalPrice']/span/text()").extract()
# time.sleep(1)
if len(jiage) > 0:
item['jiage'] = jiage[0]
else:
item['jiage'] = ""
# print(jiage)
danjia = bk.xpath(".//div[@class='unitPrice']/span/text()").extract()
# time.sleep(1)
if len(danjia) > 0:
dj = danjia[0].replace("单价", "")
item['danjia'] = dj.replace("元/平米", "")
else:
item['danjia'] = ""
# print(danjia)
biaoqian = bk.xpath(".//div[@class='tag']/span/text()").extract()
time.sleep(1)
if len(biaoqian) > 0:
item['biaoqian'] = biaoqian[0]
else:
item['biaoqian'] = ""
# print(biaoqian)
mianji = bk.xpath(".//div[@class='houseInfo']/text()").extract()
# time.sleep(1)
if len(mianji) > 0:
item['mianji'] = mianji[0]
else:
item['mianji'] = ""
# print(mianji)
yield item
n_url = "https://bj.ke.com/ershoufang/pg{}/".format(self.n_id)
if self.n_id < 100:
time.sleep(5)
yield scrapy.Request(url=n_url, dont_filter=True, callback=self.parse)
# print(self.page_id)
self.n_id += 1
二、数据分析代码:
1.饼图.html
import pymysql
from pyecharts.charts import Bar, Pie, Line
import pyecharts.options as opts
def select_huxing_jiage():
conn = pymysql.connect('localhost', 'root', '1234', 'beike')
cur = conn.cursor()
select_sql = "SELECT huxing,SUM(jiage) FROM beike_beijing GROUP BY huxing;"
cur.execute(select_sql)
result1 = cur.fetchall()
# print(result1)
huxing = []
jiage = []
for i in result1:
huxing.append(i[0])
jiage.append(int(i[1]))
return huxing, jiage
def bingtu(): # 饼图
huxing, jiage = select_huxing_jiage()
c = (
Pie()
.add("", [list(z) for z in zip(huxing, jiage)],
center=["50%", "60%"])
.set_global_opts(title_opts=opts.TitleOpts(title="户型价格销量比例图"))
.set_series_opts(label_opts=opts.LabelOpts(formatter="{b}: {c}"))
)
c.render("价格饼图.html")
if __name__ == '__main__':
bingtu()
2.柱状图.html
import pymysql
from pyecharts.charts import Bar, Pie, Line
import pyecharts.options as opts
def select_huxing_jiage():
conn = pymysql.connect('localhost', 'root', '1234', 'beike')
cur = conn.cursor()
select_sql = "SELECT huxing,SUM(jiage) FROM beike_beijing GROUP BY huxing;"
cur.execute(select_sql)
result1 = cur.fetchall()
# print(result1)
huxing = []
jiage = []
for i in result1:
huxing.append(i[0])
jiage.append(int(i[1]))
return huxing, jiage
def zhuzhuangtu(): # 柱状图
huxing, jiage = select_huxing_jiage()
bar = Bar(init_opts=opts.InitOpts(width='1000px', height='600px'))
bar.add_xaxis(huxing)
bar.add_yaxis("销量", jiage)
bar.set_global_opts(title_opts=opts.TitleOpts("各户型价格统计"))
bar.set_series_opts(label_opts=opts.LabelOpts(position="top"))
bar.render("价格柱状图.html")
if __name__ == '__main__':
zhuzhuangtu()
3.词云
from pyecharts.charts import WordCloud
import jieba
import pymysql
import wordcloud
def ciyun_beike():
conn = pymysql.connect('localhost', 'root', '1234', 'beike')
cur = conn.cursor()
select_dangdang_sql = "SELECT biaoti FROM beike_beijing;"
cur.execute(select_dangdang_sql)
beike = cur.fetchall()
beike_list = []
for i in beike:
beike_list.append(i[0])
dd_str = " ".join(beike_list)
# print(lj)
lj = wordcloud.WordCloud(font_path="词云字体.ttf", width=1000, height=1000)
lj.generate(dd_str)
lj.to_file("贝壳词云.png")
if __name__ == '__main__':
ciyun_beike()
还有其他一些数据分析图就不放了 太多了…
结果: