tensorflow学习——爬虫（图片及IP地址）

1、使用 requests 方法
百度图片的下载

import re
import requests
import os

def dowmloadPic(html,keyword):

    pic_url = re.findall('"objURL":"(.*?)",',html,re.S)
    i = 0
    print ('找到关键词:'+keyword+'的图片，现在开始下载图片...')
    for each in pic_url:
        print ('正在下载第'+str(i+1)+'张图片，图片地址:'+str(each))
        try:
            pic= requests.get(each, timeout=10)
        except requests.exceptions.ConnectionError:
            print ('【错误】当前图片无法下载')
            continue
        string = 'pictures\\'+keyword+'_'+str(i) + '.jpg'
        with open(string,'wb') as fp:
            fp.write(pic.content)
        i += 1

if __name__ == '__main__':

    session = requests.Session()
    if os.path.exists('pictures') is False:
        os.makedirs('pictures')
    word = input("Input key word: ")
#    url = 'http://image.baidu.com/search/flip?tn=baiduimage&ie=utf-8&word='+word+'&ct=201326592&v=flip'
    url = 'https://image.baidu.com/search/index?tn=baiduimage&ipn=r&ct=201326592&cl=2&lm=-1&st=-1&sf=1&fmq=&pv=&ic=0&nc=1&z=&se=1&showtab=0&fb=0&width=&height=&face=0&istype=2&ie=utf-8&fm=index&pos=history&word=' +word 
    result = session.get(url)
    dowmloadPic(result.text,word)

2、使用 urllib.request 方法

import re
import urllib.request

def open_url(url):
    req = urllib.request.Request(url)
    req.add_header("User-Agent", "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.87 Safari/537.36")

    page = urllib.request.urlopen(req)
    html = page.read().decode('utf-8')
#    print(html)
    return html
''' 
# 用于搜索网页中照片，并下载到本地   
def get_img(html):
    p = r'"objURL":"([^"]+\.jpeg)",'
#    p = r'"objURL":"(.*?)",'
#    print(html)
    iplist = re.findall(p, html)
#    print(imglist)
    for each in iplist:
        filename = each.split('/')[-1]
        print(filename)
        # 将网页保存在本地
        urllib.request.urlretrieve(each, filename, None)
'''
#用于爬取网站中存在的ip地址       
def get_img(html):
    p = r'(?:(?:[0,1]?\d?\d?|2[0-4]\d|25[0-5])\.){3}(?:[0,1]?\d?\d?|2[0-4]\d|25[0-5])'
#    p=r'(?:(?:\d\d\d|\d\d|\d)\.){3}(?:\d\d\d|\d\d|\d)'
#    print(html)
    iplist = re.findall(p, html)
#    print(imglist)
    for each in iplist:
        print(each)
        # 将网页保存在本地
#        urllib.request.urlretrieve(each, filename, None)

if __name__ == '__main__':
    url = 'http://www.xicidaili.com/'
    get_img(open_url(url))