【icourse163】学习python爬虫的代码整理

学习python爬虫的代码整理

代码思路等均参考课程：http://www.icourse163.org/learn/BIT-1001870001?tid=1002236011

1 对于一般的网页的抓取

def getHTML(url):
    try:
        response = requests.get(url,timeout = 30)
        response.raise_for_status() #如果状态不是200，则返回“产生异常”
        response.encoding = response.apparent_encoding
        return response.text
    except:
        return "产生异常"

if __name__=="__main__":
        url = "https://item.jd.com/5181380.html"
        print(getHTML(url))

2 亚马逊的例子

由于亚马逊有来源审查功能，因此限制User-Agent字段为Python的访问,所以采用以下方法爬取亚马逊商品：

def getHTMLAmazon(url):
    try:
        kv = {'User-Agent':'Mozilla/5.0'}
        response = requests.get(url, headers = kv)
        response.raise_for_status()
        response.encoding = response.apparent_encoding
        return response.text
    except:
        return "爬取失败"   


if __name__=="__main__":
        url = "https://www.amazon.cn/gp/product/B01M8L5Z3Y"
        print(getHTMLAmazon(url))

3 提供百度搜索的接口

def getBaiduso(keyword):
    try:
        kv ={"wd":keyword}
        response = requests.get("http://www.baidu.com/s", params = kv)
        print response.request.url
        response.raise_for_status()
        return len(response.text)
    except:
        return "爬取失败"

keyword = "Python"
print getBaiduso(keyword)

4 爬取图片并保存

def getPic(url,root):
    path = root +url.split('/')[-1]
    try:
        if not os.path.exists(root):
            os.mkdir(root)
        if not os.path.exists(path):
            response = requests.get(url)
            response.raise_for_status()
            with open(path,"wb") as f:
                f.write(response.content)
                f.close()
                print "文件已保存"
        else:
            print "文件已存在"
    except:
        print "爬取失败"

url = "http://image.nationalgeographic.com.cn/2017/0526/20170526025441983.jpg"
root = "E://Python dir//"
getPic(url,root)

5 查询IP地址归属地

def getIP(ipaddress):
    try:
        kv ={"ip":ipaddress}
        response = requests.get("http://m.ip138.com/ip.asp", params = kv)
        print response.request.url
        response.raise_for_status()
        return response.text[-500:]
    except:
        return "爬取失败"

ipaddress = "202.204.80.112"
print getIP(ipaddress)