爬虫|基于船讯网实现AIS数据爬虫 船讯网轨迹数据以及轨迹信息

获得所需要的MMSI

import requests
import json
# 获取 MMSI
url_MMSI = "https://www.myships.com/ms/label/getShipsBasedOnConditions"
areaId = {
    '海南': 46,
    '广西': 45, '深圳': 44, '广东': 43, '福建': 42, '浙江': 41, '上海': 40, '连云港': 39, '江苏': 38, '山东': 37, '天津': 36, '河北': 34, '辽宁': 33
}
shipSize = "50-100"
shipType = "cargo"
shipCountry = "CHN"


User_Agent = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/85.0.4183.102 Safari/537.36'
# cookie 动态改变，最好登录后复制到此处，立即爬取
Cookie = 'access_token=eyJhbGciOiJIUzI1NiJ9.eyJpYXQiOjE2MDYxMjgwNjMsImV4cCI6MTYwNjE1Njg2Mywic3ViIjoie1xuICBcImlkXCIgOiAxMDEyMixcbiAgXCJjcmVhdGVCeVwiIDogXCJhbm9ueW1vdXNVc2VyXCIsXG4gIFwiY3JlYXRlVGltZVwiIDogXCIyMDIwLTExLTIzVDA5OjEwOjEyWlwiLFxuICBcImxhc3RVcGRhdGVUaW1lXCIgOiBcIjIwMjAtMTEtMjNUMDk6MTA6MTJaXCIsXG4gIFwibmFtZVwiIDogXCJ3dWh1YWlzalwiLFxuICBcIm1vYmlsZVwiIDogXCIxNTA3MzE1MzQ3NFwiLFxuICBcInN0YXR1c1wiIDogMSxcbiAgXCJsb2dpblRpbWVcIiA6IDE2MDYxMjI2Mjk2NjYsXG4gIFwibG9naW5EZXZpY2VcIiA6IFwiUENcIixcbiAgXCJkZXB0SWRcIiA6IDIsXG4gIFwic3lzVHlwZVwiIDogMCxcbiAgXCJkZWxGbGFnXCIgOiAwLFxuICBcInJlZ1R5cGVcIiA6IDEsXG4gIFwidXNlclJvbGVzXCIgOiBbIF0sXG4gIFwiZW1haWxMaXN0XCIgOiBbIF0sXG4gIFwibWVudUR1cmF0aW9uTGlzdFwiIDogWyBdXG59In0.SYf-8CKtTR6vR2hJ--SOjt2caaAT4zilSbfG2jU9by4'
data = {'areaId': areaId['海南'], 'shipSize': shipSize,
        'shipType': shipType, 'shipCountry': shipCountry}
header = {'User-Agent': User_Agent, 'Cookie': Cookie}
req = requests.post(url=url_MMSI, headers=header, data=json.dumps(data))
with open('海南-中国-50-100-货船.json', 'w') as f:
    json.dump(json.loads(req.text), f)

爬取轨迹代码

#爬取轨迹位置，时间，速度    开始时间mmsi结束时间 构成轨迹唯一id
def run(name,header,ports):
    db = mysql.mysql()
    for startPort in ports:
        time.sleep(4)
        sql1 = "select MMSI,s,e from voyageInfo where requested is null and startPort ='"+startPort+"'"
        data1=db.select(sql1)
        if data1['msg']:
            if len(data1['res']) == 0:
                print(name+' '+startPort + " no info...")
                continue
            for i in data1['res']:
                time.sleep(4)          
                mmsi = i[0]
                s = i[1]
                e = i[2]
                print(name+' '+startPort+" now mmsi:"+mmsi)
                data2 = {'zoomleavel': 11, 'shipid': int(mmsi), 'begindate': s,
                        'enddate': e, 'common': 0, 'map_type': 'wmap', 'lang': 'ZH', 'zone': -480}
                req = requests.post(url=url_shipTrail,headers=header,data=data2)
                if req.status_code==200:
                    trailInfo = ElementTree.XML(req.text)
                    sql2 = 'insert into trailInfo(mmsi,time,location,speed,trailId) values'
                    allRes = []
                    for i in trailInfo:
                        allRes.append(i.text)
                    if allRes == 0:
                        print(name+' '+mmsi+" no trail...")
                        sql3="update voyageInfo set requested=0 where mmsi='"+mmsi+"' and s='"+s+"' and e='"+e+"'"
                        db.update(sql3)
                        continue
                    for i in range(0,len(allRes),9):
                        time1 = allRes[i]
                        x = allRes[i+1]
                        y = allRes[i+2]
                        sql2+="('"+mmsi+"','"+time1+"',ST_GeomFromText('POINT("+x+" "+y+")'),"
                        if allRes[i+5] != 'null':
                            speed = allRes[i+5]
                            sql2 += "'"+speed+"',"
                        else:
                            sql2 +="Null,"
                        trailId = s+" "+mmsi+" "+e
                        sql2+="'"+trailId+"'),"
                    if sql2 == 'insert into trailInfo(mmsi,time,location,speed,trailId) values':
                        print(name+' '+mmsi+" no trail...")
                        sql3="update voyageInfo set requested=0 where mmsi='"+mmsi+"' and s='"+s+"' and e='"+e+"'"
                        db.update(sql3)
                        continue
                    else:
                        tag = db.insert(sql2[0:-1])
                        if not tag:
                            print(sql2)
                            print(name+':insert failed ...')
                            continue
                        else:
                            sql3="update voyageInfo set requested=1 where mmsi='"+mmsi+"' and s='"+s+"' and e='"+e+"'"
                            db.update(sql3)             
                else:
                    print(name +':requested failed;cookie expired,please change new cookies...')
                    break
                
            print(startPort+' end')
        else: 
            print('select failed ...')
            print(sql1)
            continue
    print(name+' end')
    db.close()