import requests
from bs4 import BeautifulSoup
import datetime
import csv
import time
headers = {
'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.97 Safari/537.36'
}
引用该引用的库
def save_date(row):
filename = datetime.datetime.now().strftime('%Y-%m-%d_%H%M')+'.csv'
f=open(filename,'a',encoding='GBK',newline='')
csv_writer = csv.writer(f)
csv_writer.writerow(row)
定义储存过程
def resopnse_url(url):
response = requests.get(url,headers=headers)
html = BeautifulSoup(response.content,'lxml')
table = html.find(class_='resultTable5')
dates = table.find_all('tr')
for date in dates:
township = date.find_all('td')[0].text
deal_num = date.find_all('td')[1].text
area = date.find_all('td')[2].text
deal_price=date.find_all('td')[3].text
row = [township,deal_num,area,deal_price]
save_date(row)
定义解析函数
def main():
url = 'http://dgfc.dg.gov.cn/dgwebsite_v2/Secondhand/DailyStatement.aspx'
resopnse_url(url)
主函数
if __name__ == '__main__':
while True:
now = datetime.datetime.now()
if now.hour == 23 and now.minute == 55:
main()
time.sleep(60)
程序入口,每天23点55分的时候,进行网站爬取
东莞房管局网站
http://dgfc.dg.gov.cn/dgwebsite_v2/Secondhand/DailyStatement.aspx