爬取携程旅游攻略

 获取所有攻略的接口

import requests

proxy = {
    "http":"",
    "https":""
}
headers ={
"Accept":"*/*",
"Accept-Encoding":"gzip, deflate, br",
"Accept-Language":"zh-CN,zh;q=0.9",
"Cache-Control":"no-cache",
"Pragma":"no-cache",
"Referer":'https://you.ctrip.com/',
'Sec-Ch-Ua':'"Google Chrome";v="119", "Chromium";v="119", "Not?A_Brand";v="24"',
'Sec-Ch-Ua-Mobile':'?0',
'Sec-Ch-Ua-Platform':'Windows',
'Sec-Fetch-Dest':'empty',
'Sec-Fetch-Mode':'cors',
'Sec-Fetch-Site':'same-site',
'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36',
'Content-Type':'application/json',
'Cookieorigin':'https://you.ctrip.com',
'Origin':'https://you.ctrip.com',
}


n = "09031118216454371269"
st = str(int(time.time()*1000))
rd = ''.join(random.sample("1234567890",7))
t =f"{n}-{st}-{rd}"

url = f"https://m.ctrip.com/restapi/soa2/22670/getRecommendTravel?_fxpcqlniredt={n}&x-traceID={t}"
data = {"head":{"cid":n,"ctok":"","cver":"1.0","lang":"01","sid":"8888","syscode":"999","auth":"","xsid":"","extension":[]},"districtId":61,"sourceFrom":0,"type":3,"pageIndex":2}

res = requests.post(url=url,headers=headers,proxies=proxy,json=data)
print(res.text)

{"responseStatus":{"timestamp":"/Date(1699933495659+0800)/","ack":"Success","errors":[],"extension":[{"id":"CLOGGING_TRACE_ID","value":"6189775829785536957"},{"id":"RootMessageId","value":"100025527-0a70c299-472203-1924391"}]},"result":0,"pageCount":70,"travelInfoList":[{"id":3876745,"title":"四天三晚，带你玩转不一样的三亚","author":{"avatar":"https://dimg04.c-ctrip.com/images/Z80t14000000wn2bzE8E4_C_180_180.jpg","name":"滢萱"},"tagList":[4,5],"content":"一：三亚只有海？你OUT啦！你心中的 三亚 ，是不是这样？其实啊， 三亚 不仅仅只有","pictureNumber":"273","commentNumber":"12","viewNumber":"53568","imageUrl":"https://youimg1.c-ctrip.com/target/100r1600000101ayuAA1E_D_230_160.jpg","jumpUrl":"http://you.ctrip.com/travels/Sanya61/3876745.html"},{"id":3975829,"title":"完美错过疫情的三亚亲子假期超全攻略（四家丰俭由人的酒店及美食攻略）","author":{"avatar":"https://dimg04.c-ctrip.com/images/Z8090j000000a7nhw85FC_C_180_180.jpg","name":"飘女孩儿"},"tagList":[1,5],"content":"超幸运的我们在 北京 的第一波疫情和第二波疫情期间出行了，今年出境游太难，所以我们的","pictureNumber":"370","commentNumber":"13","viewNumber":"13494","imageUrl":"https://youimg1.c-ctrip.com/target/0101c1200086e2uhb51CC_D_230_160.jpg","jumpUrl":"http://you.ctrip.com/travels/Sanya61/3975829.html"},{"id":3918978,"title":"一封关于三亚的冬季来信，请各位查收","author":{"avatar":"https://dimg04.c-ctrip.com/images/0Z830120008css4h4C747_C_180_180.jpg","name":"奔跑的小东东"},"tagList":[1,5],"content":"一：自驾三亚，去探寻不一样的时光曾经沧海难为水，除却巫山不是云。如果你来过三亚，那么","pictureNumber":"218","commentNumber":"32","viewNumber":"112980","imageUrl":"https://youimg1.c-ctrip.com/target/100c1b000001a17v71A00_D_230_160.jpg","jumpUrl":"http://you.ctrip.com/travels/Sanya61/3918978.html"},{"id":3998789,"title":"三亚自驾之旅，感受碧海蓝天，亚特兰蒂斯做一天美人鱼","author":{"avatar":"https://dimg04.c-ctrip.com/images/fd/headphoto/g2/M03/7A/2C/Cghzf1VwW8aAEcE4AACpkZ2V0m0688_C_180_180.jpg","name":"何陈文"},"tagList":[1,5],"content":"旅行.序言2020年真的是不平凡的一年，突如其来的疫情真的改变了我们很多。在家宅了几","pictureNumber":"239","commentNumber":"19","viewNumber":"12036","imageUrl":"https://dimg04.c-ctrip.com/images/0103l120008h027kt0B63_D_230_160.jpg","jumpUrl":"http://you.ctrip.com/travels/Sanya61/3998789.html"},{"id":3967210,"title":"三亚自驾之旅，玩转碧海蓝天，想念三亚的美食","author":{"avatar":"https://dimg04.c-ctrip.com/images/fd/headphoto/g2/M03/7A/2C/Cghzf1VwW8aAEcE4AACpkZ2V0m0688_C_180_180.jpg","name":"何陈文"},"tagList":[1,5],"content":"旅行.序言旅行，对于我来说，是生活的调味品，旅行的意义在于让自己的心境更淡然，把之前","pictureNumber":"330","commentNumber":"23","viewNumber":"36599","imageUrl":"https://youimg1.c-ctrip.com/target/0105d1200082cfoigF2FE_D_230_160.jpg","jumpUrl":"http://you.ctrip.com/travels/Sanya61/3967210.html"},{"id":3970548,"title":"感受椰风海韵的海南岛，寻找东线最美观海点，亲子游记","author":{"avatar":"https://dimg04.c-ctrip.com/images/Z80g0u000000ix2wlFBF6_C_180_180.jpg","name":"乐玩日志"},"tagList":[1,5],"content":"前言北纬18度，是地球上最神奇的地理区域之一，之所以说它神奇， 原因就在于这个区域是","pictureNumber":"339","commentNumber":"15","viewNumber":"122884","imageUrl":"https://youimg1.c-ctrip.com/target/010011200083z3u6w331B_D_230_160.jpg","jumpUrl":"http://you.ctrip.com/travels/Sanya61/3970548.html"},{"id":3989893,"title":"环岛自驾，去海南赴一个碧海蓝天","author":{"avatar":"https://dimg04.c-ctrip.com/images/fd/headphoto/g2/M07/44/19/CghzgVSS6BOAUa6HAAEHat8lBLk630_C_180_180.jpg","name":"Chen浩文"},"tagList":[3,5],"content":"【写在开始】世俗的纷扰，生活的琐碎使人精疲力尽，黑夜下，撕开那张面具尽是怠倦的容颜，","pictureNumber":"154","commentNumber":"19","viewNumber":"31102","imageUrl":"https://dimg04.c-ctrip.com/images/0105g120008cjuyiw1CA8_D_230_160.jpg","jumpUrl":"http://you.ctrip.com/travels/Haikou37/3989893.html"},{"id":3981953,"title":"疫情过后 尽情享受大海与美食的三亚放飞之旅","author":{"avatar":"https://dimg04.c-ctrip.com/images/Z80m12000000si70j1891_C_180_180.jpg","name":"香香花猫"},"tagList":[1,5],"content":"疫情把我们困在家里大半年，基本在家的日子是数着手指头度过的，无法出门对于热爱旅行的人","pictureNumber":"249","commentNumber":"23","viewNumber":"12282","imageUrl":"https://youimg1.c-ctrip.com/target/0102s1200089sxx3k342C_D_230_160.jpg","jumpUrl":"http://you.ctrip.com/travels/Sanya61/3981953.html"},{"id":3810908,"title":"暑假带娃去三亚，2019年最全三亚亲子游攻略","author":{"avatar":"https://dimg04.c-ctrip.com/images/Z80h0n000000ec5sw1F0A_C_180_180.jpg","name":"JOJO带你游世界"},"tagList":[1,5],"content":"对于带娃旅行这件全球公认的最坑爹业务，我相信每位爹妈都会有很多牢骚想发，甚至脑海中还","pictureNumber":"344","commentNumber":"44","viewNumber":"43314","imageUrl":"https://youimg1.c-ctrip.com/target/100514000000x04l94A67_D_230_160.jpg","jumpUrl":"http://you.ctrip.com/travels/Sanya61/3810908.html"},{"id":3930674,"title":"❤漫步三亚❤愿做一只候鸟~飞往北纬18°过冬","author":{"avatar":"https://dimg04.c-ctrip.com/images/headphoto/434/463/096/f1846bb807f14b6ab550f463e27bddb2_C_180_180.jpg","name":"雍容makiori"},"tagList":[1,5],"content":"你在南方的艳阳里，四季如春……我在北方的雾霾里，生不如死……我清晰的记住当我飞机落地","pictureNumber":"328","commentNumber":"21","viewNumber":"10254","imageUrl":"https://dimg04.c-ctrip.com/images/100g1c000001d85qrBED6_D_230_160.jpg","jumpUrl":"http://you.ctrip.com/travels/Sanya61/3930674.html"}]}

 提取每一个攻略的详细信息

from lxml import etree
rep = requests.get(url="https://you.ctrip.com/travels/Sanya61/3969175.html",headers=headers,proxies=proxy)
html=etree.HTML(rep.text)
title = html.xpath("//div[@class='ctd_head_left']/h2//text()")

print(title)
fun_info = html.xpath("//div[@class='ctd_content_controls cf']/div/span//text()")
print(fun_info)

place = html.xpath("//div[@class='author_poi']/dl/dd/a//text()")
print(place)

conten_text = html.xpath("//div[@class='ctd_content']/p//text()")
print(conten_text)

img_list = html.xpath("//div[@class='content cf']//div[@class='img']/a/@href")
print(img_list)
for i,url in enumerate(img_list):
    res = requests.get(url=url)
    with open(f'./img/{i}.jpg','wb') as f:
        f.write(res.content)