Python页面爬虫学习1-基础中的基础

import requests
'''
re = requests.get("http://www.baidu.com")
re.encoding = re.apparent_encoding
re.status_code
print(re.status_code)
print(re.text)
print(re.headers)
print(re.apparent_encoding)
print(re.content)
'''
'''
网页爬虫的通用代码框架
'''
def getHTMLText(url,kv):
    try:
        kv_borswer_U-Agent= {'User-Agent':'Mozzila/5'}# 更改发送的user-Agent,避免网站规避爬虫
        r = requests.get(url,params=kv,headers=kv_borswer_U-Agent)
        print(r.request.headers)
        print(r.request.url)
        print(len(r.text))
       # print(r.raise_for_status())
        r.raise_for_status()
        r.encoding  = r.apparent_encoding
        return r.text
    except:
        print(r.status_code)
        return "产生异常"
kv_baidu_word_search = {"wd":"测试"}
kv_baidu_image_word = {"word":"测试"}
kv_jd_search_keyword ={"keyword":"手机","ev":"exbrand_小米"}
kv_tmall_search_q ={"q":"手机"}

'''
百度图片搜索链接 ----- https://image.baidu.com/search/index?tn=baiduimage   word
京东搜索 ----- https://search.jd.com/Search?keyword=%E6%89%8B%E6%9C%BA&enc=utf-8&wq=%E6%89%8B%E6%9C%BA&pvid=0661a6aa442d4d61b5b97f50fbb849b0
天猫搜索 ----- https://list.tmall.com/search_product.htm
'''
print(getHTMLText("http://www.baidu.com/s",kv_baidu_word_search))
print(getHTMLText("https://image.baidu.com/search/index?tn=baiduimage",kv_baidu_image_word))
print(getHTMLText("https://search.jd.com/Search",kv_jd_search_keyword))
print(getHTMLText("https://list.tmall.com/search_product.htm",kv_tmall_search_q))

后续需要持续整理相关的页面，便于后期数据整理