python爬取网页的代码_Python爬取网页全代码

#爬取京东全代码

import requests

url = "http://item.jd.com/2967929.html"

try:

r = requests.get(url)

r.raise_for_status() #在返回200不产生异常，否则会产生异常

r.encoding = r.apparent_encoding

print(r.text[:10000])

except:

print("爬取失败")

#爬取亚马逊全代码

import requests

url = "https://www.amazon.cn/gp/product/B01M8L5Z3Y"

try:

kv = {‘user-agent‘:‘Mozilla/5.0‘}

r = requests.get(url,headers=kv)

r.raise_for_status() #在返回200不产生异常，否则会产生异常

r.encoding = r.apparent_encoding

print(r.text[1000:2000])

except:

print("爬取失败")

#爬取百度搜索全代码

import requests

keyword = "Python"

try:

kv = {‘wd‘:keyword}

r = requests.get("http://www.baidu.com/s",params=kv)

print(r.request.url)

r.raise_for_status()

print(len(r.text))

except:

print("爬取失败")

#爬取360搜索全代码

import requests

keyword = "Python"

try:

kv = {‘q‘:keyword}

r = requests.get("http://www.so.com/s",params=kv)

print(r.request.url)

r.raise_for_status()

print(len(r.text))

except:

print("爬取失败")

原文地址：https://www.cnblogs.com/zhenshj/p/9041027.html