链家二手房在售爬虫

最新推荐文章于 2024-09-12 07:51:58 发布

weixin_43139613

最新推荐文章于 2024-09-12 07:51:58 发布

阅读量535

点赞数

分类专栏：笔记 python基础文章标签： Python

本文链接：https://blog.csdn.net/weixin_43139613/article/details/88807584

版权

笔记同时被 2 个专栏收录

105 篇文章 1 订阅

订阅专栏

python基础

52 篇文章 0 订阅

订阅专栏

# -*- coding: utf-8 -*-
"""
Created on Mon Mar 25 21:13:13 2019

@author: 盗号
"""

# -*- coding: utf-8 -*-
"""
Created on Mon Mar 25 10:10:22 2019

@author: zhang-peng
"""
# -*- coding: utf-8 -*-
import bs4 
import requests
import time#引入time，计算下载时间


def open_url(url):
    hd = {}
    hd['User-Agent'] = 'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.109 Safari/537.36'
    r = requests.get(url,headers=hd,timeout=10)
    return r

host = 'https://ty.lianjia.com/ershoufang/pg'

whvj = []
ee = []
aa = []
bb = []
cc = []
dd = []
count = 0

start = time.time()
size = 0
q = 100 


while count < q:
    url = host + str(count)
    r = open_url(url)
    soup = bs4.BeautifulSoup(r.text,'html.parser')
    count = count + 1
    
    
    targets = soup.find_all('a',class_="img")
    
    for each in targets:
        whvj.append(each['href'])
    
    print('\r'+"已经下载："+int(count/q*100)*"█"+"【"+str(round(float(count/q)*100,2))+"%"+"】",end="")
#    print(url)
#    print(whvj)
whvj = list(set(whvj))
count1 = 0 
response = requests.get(url,stream = True)#stream参数设置成True时，它不会立即开始下载，当你使用iter_content或iter_lines遍历内容或访问内容属性时才开始下载
chunk_size = 1024#每次块大小为1024
content_size = int(len(whvj))
for i in whvj:
    soup1 = bs4.BeautifulSoup(open_url(i).text,'html.parser')
    title = soup1.find_all('h1',class_='main')
    aa.append(title[0].text.split())
    xbxi = soup1.find_all("div",class_="content")

    bb.append(xbxi[2].text.split())
#    cc.append(xbxi[3].text.split())
    dd.append(soup1.find_all("span",class_="info")[0].text.split())
    jxge = soup1.find_all('div',class_='price')
    for i in jxge:
        ee.append(i.text)
    size = size +1 
    print('\r'+"已经下载："+int(size/content_size*100)*"█"+" 【"+str(round(size/chunk_size/1024,2))+"MB】"+"【"+str(round(float(size/content_size)*100,2))+"%"+"】",end="")

##
result = []
length =len(whvj)
for i in range(length):
    result.append(str(dd[i])+'^'  +str(aa[i])+'^'+str(whvj[i])+ '^' + str(ee[i])
                    + '^' + str(bb[i]) + '^' +  '\n')
    
end = time.time()
print("总耗时:"+str(end-start)+"秒")

#data_count = 0

with open('ty.txt','w',encoding='utf-8') as f:
    for each in result:
        f.write(each)