【爬虫练手】东莞当天二手成交信息爬虫

本文链接：https://blog.csdn.net/wyh33200/article/details/106791710

import requests
from bs4 import BeautifulSoup
import datetime
import csv
import time
headers = {
    'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.97 Safari/537.36'
}

引用该引用的库

def save_date(row):
    filename = datetime.datetime.now().strftime('%Y-%m-%d_%H%M')+'.csv'
    f=open(filename,'a',encoding='GBK',newline='')
    csv_writer = csv.writer(f)
    csv_writer.writerow(row)

定义储存过程

def resopnse_url(url):
    response = requests.get(url,headers=headers)
    html = BeautifulSoup(response.content,'lxml')
    table = html.find(class_='resultTable5')
    dates = table.find_all('tr')
    for date in dates:
        township = date.find_all('td')[0].text
        deal_num = date.find_all('td')[1].text
        area = date.find_all('td')[2].text
        deal_price=date.find_all('td')[3].text
        row = [township,deal_num,area,deal_price]
        save_date(row)

定义解析函数

def main():
    url = 'http://dgfc.dg.gov.cn/dgwebsite_v2/Secondhand/DailyStatement.aspx'
    resopnse_url(url)

主函数

if __name__ == '__main__':
    while True:
        now = datetime.datetime.now()
        if now.hour == 23 and now.minute == 55:
            main()
            time.sleep(60)