每天定时采集（当前时间到月底）携程机票数据

最新推荐文章于 2024-07-19 15:37:43 发布

置顶 Andy Chen 陈郑游

最新推荐文章于 2024-07-19 15:37:43 发布

阅读量1.4k

点赞数

分类专栏： ————[ Python ] 文章标签： Python 携程机票 Python采集数据 Python定时采集 schedule定时任务

本文链接：https://blog.csdn.net/JavaWebRookie/article/details/102792685

版权

————[ Python ] 专栏收录该内容

3 篇文章 2 订阅

订阅专栏

每天定时早上八点、下午三点、晚上八点采集数据

if __name__ == "__main__":
    schedule.every().day.at('08:00').do(saveData)
    schedule.every().day.at('15:00').do(saveData)
    schedule.every().day.at('20:00').do(saveData)
    while True:
        schedule.run_pending()

classType 类型、point 采集的城市

def saveData():
    classType = ["经济舱","公务舱/头等舱"]
    point = ["北京","上海","哈尔滨", "成都"]
    # 本月当天时间 
    nowTime = datetime.datetime.now()
    nowHour = nowTime.strftime('%Y-%m-%d %H')+":00:00"
    begin = datetime.date(nowTime.year,nowTime.month,nowTime.day)
    # 本月最后一天
    dayEnd = getMonthFirstDayAndLastDay(nowTime.year,nowTime.month).day
    end = datetime.date(nowTime.year,nowTime.month,dayEnd)

    d = begin
    delta = datetime.timedelta(days=1)
    while d <= end:
        for cltype in classType:
            for startP in point:
                xiecheng("起点城市", startP, str(d) ,nowHour,cltype)
                time.sleep(30)
        
            for endP in point:
                xiecheng(endP, "终点城市", str(d) ,nowHour,cltype)
                time.sleep(30)   
        
        d += delta

def getMonthFirstDayAndLastDay(year=None, month=None):
    """
    :param year: 年份，默认是本年，可传int或str类型
    :param month: 月份，默认是本月，可传int或str类型
    :return: firstDay: 当月的第一天，datetime.date类型
              lastDay: 当月的最后一天，datetime.date类型
    """
    if year:
        year = int(year)
    else:
        year = datetime.date.today().year

    if month:
        month = int(month)
    else:
        month = datetime.date.today().month

    # 获取当月第一天的星期和当月的总天数
    firstDayWeekDay, monthRange = calendar.monthrange(year, month)

    # 获取当月的第一天
    # firstDay = datetime.date(year=year, month=month, day=1)
    lastDay = datetime.date(year=year, month=month, day=monthRange)
    return  lastDay

from prettytable import PrettyTable
import requests
import json
import time
import datetime
import calendar
import def_xc_db
import random
import schedule


def xiecheng( dcity, acity, date, activeDate,classType):
    UserAgent = [
			"Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/66.0.3359.139 Safari/537.36",
			"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/535.7 (KHTML, like Gecko) Chrome/16.0.912.36 Safari/535.7",
			"Mozilla/5.0 (Windows NT 6.2; Win64; x64; rv:16.0) Gecko/16.0 Firefox/16.0",
			"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_3) AppleWebKit/534.55.3 (KHTML, like Gecko) Version/5.1.3 Safari/534.53.10",
			"Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Win64; x64; Trident/5.0; .NET CLR 3.5.30729; .NET CLR 3.0.30729; .NET CLR 2.0.50727; Media Center PC 6.0)",
			"Mozilla/5.0 (compatible; MSIE 8.0; Windows NT 6.0; Trident/4.0; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; .NET CLR 1.0.3705; .NET CLR 1.1.4322)",
			"Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/27.0.1500.55 Safari/537.36",
			"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_2) AppleWebKit/537.17 (KHTML, like Gecko) Chrome/24.0.1309.0 Safari/537.17"
			"Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:2.0b13pre) Gecko/20110307 Firefox/4.0b13pre",
			"Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:16.0) Gecko/20100101 Firefox/16.0",
			"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.64 Safari/537.11"]
    randomIndex =  random.sample(range(0,10),10)[0]
    userAgent = UserAgent[int(randomIndex)]
    headers = {
        "User-Agent":userAgent,
        "Content-Type": "application/json",  # 声明文本类型为 json 格式,
        'Cookie': '__DAYU_PP=fbffA7JMjfFQYEJvq3yV285fa84ef3bb; _abtest_userid=14f10ca5-5fe5-4b90-8994-a9a2a8289725; _ga=GA1.2.1823657567.1547218198; _RSG=jB4_phNRhw9fNvhoLy8_29; _RDG=28f1ebd8af073928c800c3c3726b485bce; _RGUID=9eab6d7a-f3fc-42ed-90bf-23c572af95e9; _geoinfo=CN%26%e6%b7%b1%e5%9c%b3; _bfa=1.1547218195619.296wjo.1.1547625090247.1554179025352.5.60; _bfs=1.1; Hm_lvt_cdce8cda34e84469b1c8015204129522=1554179026; Hm_lpvt_cdce8cda34e84469b1c8015204129522=1554179026; gad_city=31f35a60e938dff697ddea628b5bea7c; _RF1=14.153.237.134; Session=SmartLinkCode=U155952&SmartLinkKeyWord=&SmartLinkQuary=&SmartLinkHost=&SmartLinkLanguage=zh; _gid=GA1.2.1875696831.1554179028; _gat=1; Mkt_UnionRecord=%5B%7B%22aid%22%3A%224897%22%2C%22timestamp%22%3A1554179028552%7D%5D; _jzqco=%7C%7C%7C%7C1547625095764%7C1.1997966906.1547218199380.1547636825575.1554179028593.1547636825575.1554179028593.0.0.0.59.59; __zpspc=9.5.1554179028.1554179028.1%232%7Csp0.baidu.com%7C%7C%7C%25E6%2590%25BA%25E7%25A8%258B%7C%23; MKT_Pagesource=PC; Union=OUID=index&AllianceID=4897&SID=155952&SourceID=&Expires=1554783828996; MKT_OrderClick=ASID=4897155952&CT=1554179029004&CURL=https%3A%2F%2Fwww.ctrip.com%2F%3Fsid%3D155952%26allianceid%3D4897%26ouid%3Dindex&VAL={"pc_vid":"1547218195619.296wjo"}; _bfi=p1%3D100101991%26p2%3D100101991%26v1%3D60%26v2%3D59'
    }
 
    city = { '阿尔山': 'YIE', '阿克苏': 'AKU', '阿拉善右旗': 'RHT', '阿拉善左旗': 'AXF', '阿勒泰': 'AAT', '阿里': 'NGQ', '澳门': 'MFM',
            '安庆': 'AQG', '安顺': 'AVA', '鞍山': 'AOG', '巴彦淖尔': 'RLK', '百色': 'AEB', '包头': 'BAV', '保山': 'BSD', '北海': 'BHY',
            '北京': 'BJS', '白城': 'DBC', '白山': 'NBS', '毕节': 'BFJ', '博乐': 'BPL', '重庆': 'CKG', '昌都': 'BPX', '常德': 'CGD',
            '常州': 'CZX', '朝阳': 'CHG', '成都': 'CTU', '池州': 'JUH', '赤峰': 'CIF', '揭阳': 'SWA', '长春': 'CGQ', '长沙': 'CSX',
            '长治': 'CIH', '承德': 'CDE', '沧源': 'CWJ', '达县': 'DAX', '大理': 'DLU', '大连': 'DLC', '大庆': 'DQA', '大同': 'DAT',
            '丹东': 'DDG', '稻城': 'DCY', '东营': 'DOY', '敦煌': 'DNH', '芒市': 'LUM', '额济纳旗': 'EJN', '鄂尔多斯': 'DSN', '恩施': 'ENH',
            '二连浩特': 'ERL', '佛山': 'FUO', '福州': 'FOC', '抚远': 'FYJ', '阜阳': 'FUG', '赣州': 'KOW', '格尔木': 'GOQ', '固原': 'GYU',
            '广元': 'GYS', '广州': 'CAN', '贵阳': 'KWE', '桂林': 'KWL', '哈尔滨': 'HRB', '哈密': 'HMI', '海口': 'HAK', '海拉尔': 'HLD',
            '邯郸': 'HDG', '汉中': 'HZG', '杭州': 'HGH', '合肥': 'HFE', '和田': 'HTN', '黑河': 'HEK', '呼和浩特': 'HET', '淮安': 'HIA',
            '怀化': 'HJJ', '黄山': 'TXN', '惠州': 'HUZ', '鸡西': 'JXA', '济南': 'TNA', '济宁': 'JNG', '加格达奇': 'JGD', '佳木斯': 'JMU',
            '嘉峪关': 'JGN', '金昌': 'JIC', '金门': 'KNH', '锦州': 'JNZ', '嘉义': 'CYI', '西双版纳': 'JHG', '建三江': 'JSJ', '晋江': 'JJN',
            '井冈山': 'JGS', '景德镇': 'JDZ', '九江': 'JIU', '九寨沟': 'JZH', '喀什': 'KHG', '凯里': 'KJH', '康定': 'KGT', '克拉玛依': 'KRY',
            '库车': 'KCA', '库尔勒': 'KRL', '昆明': 'KMG', '拉萨': 'LXA', '兰州': 'LHW', '黎平': 'HZH', '丽江': 'LJG', '荔波': 'LLB',
            '连云港': 'LYG', '六盘水': 'LPF', '临汾': 'LFQ', '林芝': 'LZY', '临沧': 'LNJ', '临沂': 'LYI', '柳州': 'LZH', '泸州': 'LZO',
            '洛阳': 'LYA', '吕梁': 'LLV', '澜沧': 'JMJ', '龙岩': 'LCX', '满洲里': 'NZH', '梅州': 'MXZ', '绵阳': 'MIG', '漠河': 'OHE',
            '牡丹江': 'MDG', '马祖': 'MFK', '南昌': 'KHN', '南充': 'NAO', '南京': 'NKG', '南宁': 'NNG', '南通': 'NTG', '南阳': 'NNY',
            '宁波': 'NGB', '宁蒗': 'NLH', '攀枝花': 'PZI', '普洱': 'SYM', '齐齐哈尔': 'NDG', '黔江': 'JIQ', '且末': 'IQM', '秦皇岛': 'BPE',
            '青岛': 'TAO', '庆阳': 'IQN', '衢州': 'JUZ', '日喀则': 'RKZ', '日照': 'RIZ', '三亚': 'SYX', '厦门': 'XMN', '上海': 'SHA',
            '深圳': 'SZX', '神农架': 'HPG', '沈阳': 'SHE', '石家庄': 'SJW', '塔城': 'TCG', '台州': 'HYN', '太原': 'TYN', '扬州': 'YTY',
            '唐山': 'TVS', '腾冲': 'TCZ', '天津': 'TSN', '天水': 'THQ', '通辽': 'TGO', '铜仁': 'TEN', '吐鲁番': 'TLQ', '万州': 'WXN',
            '威海': 'WEH', '潍坊': 'WEF', '温州': 'WNZ', '文山': 'WNH', '乌海': 'WUA', '乌兰浩特': 'HLH', '乌鲁木齐': 'URC', '无锡': 'WUX',
            '梧州': 'WUZ', '武汉': 'WUH', '武夷山': 'WUS', '西安': 'SIA', '西昌': 'XIC', '西宁': 'XNN', '锡林浩特': 'XIL',
            '香格里拉(迪庆)': 'DIG',
            '襄阳': 'XFN', '兴义': 'ACX', '徐州': 'XUZ', '香港': 'HKG', '烟台': 'YNT', '延安': 'ENY', '延吉': 'YNJ', '盐城': 'YNZ',
            '伊春': 'LDS',
            '伊宁': 'YIN', '宜宾': 'YBP', '宜昌': 'YIH', '宜春': 'YIC', '义乌': 'YIW', '银川': 'INC', '永州': 'LLF', '榆林': 'UYN',
            '玉树': 'YUS',
            '运城': 'YCU', '湛江': 'ZHA', '张家界': 'DYG', '张家口': 'ZQZ', '张掖': 'YZY', '昭通': 'ZAT', '郑州': 'CGO', '中卫': 'ZHY',
            '舟山': 'HSN',
            '珠海': 'ZUH', '遵义(茅台)': 'WMT', '遵义(新舟)': 'ZYI'}
 
    classTypeData = {"经济舱":"Y","公务舱/头等舱":"CF"}

    url = 'https://flights.ctrip.com/itinerary/api/12808/products'
    request_payload = {"flightWay": "Oneway",
                       "army": "false",
                       "hasChild": 'false',
                       "hasBaby": 'false',
                       "searchIndex": 1,
                       "portingToken": "d2c78cdac43e787f65b08716dd63dfd3",
                       "classType":classTypeData.get(classType),
                       "airportParams": [
                           {"dcity": city.get(dcity), "acity": city.get(acity), "dcityname": dcity, "acityname": acity,
                            "date": date}]}
 
    # 这里传进去的参数必须为 json 格式
    response = requests.post(url, data=json.dumps(request_payload), headers=headers).text
    routeList = json.loads(response)["data"].get('routeList')
    table = PrettyTable(["Airline","FlightNumber","DepartureDate" ,"ArrivalDate","PunctualityRate","LowestPrice","activeDate","filghtDate","fromCity","toCity","classType"])
    print("输出：",  json.loads(response)["status"])

    if routeList !="NoneType" :
        for route in routeList:
            if len(route.get('legs')) == 1:
                info = {}
                legs = route.get('legs')[0]
                flight = legs.get('flight')
                info['Airline'] = flight.get('airlineName')
                info['FlightNumber'] = flight.get('flightNumber')
                info['DepartureDate'] = flight.get('departureDate')[-8:-3]
                info['ArrivalDate'] = flight.get('arrivalDate')[-8:-3]
                info['PunctualityRate'] = flight.get('punctualityRate')
                info['LowestPrice'] = legs.get('characteristic').get('lowestPrice')
                info['activeDate'] = activeDate
                info['filghtDate'] = str(date)
                info['fromCity'] = dcity
                info['toCity'] = acity
                info['classType'] = classType

                # def_xc_db.updateOrSaveData(False,info) 添加到数据库操作
                table.add_row(info.values())

        print(dcity, '------->', acity, date)
        print(table)