爬虫草稿

from urllib import request
from bs4  import BeautifulSoup
import logging, os


base_url = 'http://www.xgyw.cc'
url_list = [base_url+'/Xgyw']
h_list = []
path = r'E:\python\0425\pics'
for i in range(2,5):
    url_list.append('http://www.xgyw.cc/Xgyw/page_%s.html' %i)
print(url_list)
headers={'User-Agent':'Mozilla/5.0(WindowsNT6.1;WOW64)AppleWebKit/537.36(KHTML, likeGecko) Chrome/57.0.2987.110 Safari/537.36'}

#解析网址
def get_Hrefs():
    maxtrynum = 5
    global hrefs_list
    hrefs_list = []
    for url in url_list:
        print('解析页：[%s]' % url)
        for tries in range(maxtrynum):
            try:
                req = request.Request(url=url, headers=headers)
                res = request.urlopen(req)
                html = res.read().decode('UTF-8', 'ignore').replace(u'\0xd0', '')
              # html = res.read().decode('gbk', 'ignore')
                # print(html)                                                   
                soup = BeautifulSoup(html, 'html.parser')                        
                # hrefs = soup.find_all('div', class_='biank1')                 
                hrefs = soup.select('a[href^="/Xgyw/Xgyw"]')                    
                # print(hrefs)                                                  
                for each_href in hrefs:
                    hre = each_href.get('href')
                    # print(hre)
                    hrefs_list.append(base_url + hre)
                break
            except:
                if tries < (maxtrynum - 1):
                    continue
                else:
                    logging.error("Has tried %d times to access url %s, all failed!", maxtrynum, url)
                    break
    return hrefs_list

# 列表去重
def dedupe(list):
    global L
    L=[]
    for i in list:
        if i not in L:
            yield i
            L.append(i)
    return L
# 解析page图片页
def get_pages_hrefs(href):
    times = 5
  # for href in href_list:
    for t in range(times):
        try:
            print('找到页面：%s' % href)
            req = request.Request(url=href, headers=headers)
            res = request.urlopen(req)
            html = res.read().decode('UTF-8', 'ignore').replace(u'\0xd0', '')
          # print(html)
            soup = BeautifulSoup(html, 'html.parser')
            pages = soup.select('a[href^="/Xgyw/Xgyw"]')
          # print(pages)
            for each in pages:
                addr = each.get('href')
               # print(addr)
                h_list.append('http://www.xgyw.cc' + addr)
            break
        except:
            if t < (times - 1):
                continue
            else:
                logging.error("Has tried %d times to access url %s, all failed!", times, href)
                break
    return h_list

# 解析图片&保存图片
def parser_pics(list):
    n = 5
    for each_list in list:
        for t in range(n):
            try:
                print('解析图片地址：%s' % each_list)
                req = request.Request(url=each_list, headers=headers)
                res = request.urlopen(req)
                html = res.read().decode('UTF-8', 'ignore').replace(u'\0xd0', '')
                # print(html)
                soup = BeautifulSoup(html, 'html.parser')
                pics = soup.select('img[src^="/uploadfile"]')
                #print(pics)
                for each_pic in pics:
                    srcs = each_pic.get('src')
                    print("解析的图片url:", base_url + srcs)
                    save_pics(base_url + srcs)
                break
            except:
                if t < (n - 1):
                    continue
                else:
                    logging.error("Has tried %d times to access url %s, all failed!", n, each_list)
                    break

# 保存图片
def save_pics(pic):
    fileName = path + os.sep + pic.split("/")[-1]
    if not os.path.exists(fileName):
        with open(fileName, "wb") as f:
            print("正在保存:", fileName)
            f.write(request.urlopen(pic).read())



if __name__=='__main__':
    for i in dedupe(get_Hrefs()):
        get_pages_hrefs(i)
    print(h_list)
    parser_pics(h_list)