python获取数据网页数据并创建文件夹保存（基于python3.6）

from urllib.parse import urljoin
import urllib.request

from bs4 import BeautifulSoup


import os
import datetime
import re
import errno


def mkdir_p(path):   #递归创建多级目录
    try:
        os.makedirs(path)
    except OSError as exc: # Python >2.5 (except OSError, exc: for Python <2.5)
        if exc.errno == errno.EEXIST and os.path.isdir(path):
            pass
        else: raise

def get_link(page):  # 寻找链接的href
    linkData = []
    for page in page.find_all('td'):
        links = page.select("a")
        for each in links:
            # if str(each.get('href'))[:1] == '/': 过滤if代码
                data=each.get('href')
                linkData.append(data)
    return(linkData)


def gain(url):      #获取网页指定内容
    page = urllib.request.urlopen(url).read()
    soup = BeautifulSoup(page, 'lxml')  #利用soup获取网页内容
    links = get_link(soup) #获取<a href= ? 内容
    return links


def main():
    url = 'https://www.tide-forecast.com/countries/China'
    Web_Link=gain(url)

    for Link in range(len(Web_Link)):
        Link_Add = Web_Link[Link]
        Link_One = re.split("/", Link_Add)   #去除'/'，将Link_Add变成数组
        Link_Address = Link_One[2]           #获取数组第3位值
        Link_Address = (Link_Address + '.js')
        url_Tide = 'https://www.tide-forecast.com/tides/'
        connet = urljoin(url_Tide, Link_Address)          # 拼接网址路径
        file = os.path.join('D:\\TideData\\China' + "/" )  # 拼接绝对路径
        mkdir_p(file)
        print(connet)
        if os.path.isfile(file):
           print('文件已存在')
        else:
            start = datetime.datetime.now().replace(microsecond=0)  #计时工具

            url = connet
            wp = urllib.request.urlopen(url)  #打开数据网页数据
            content = wp.read()                
            fp = open(file + Link_Address, "wb")     #写入指定文件夹 
            fp.write(content)            #写入数据
            fp.close()                    #关闭文件

            end = datetime.datetime.now().replace(microsecond=0)
            print("用时: ", end='')
            print(end - start)





if __name__ == '__main__':
    main()

转载于:https://www.cnblogs.com/setname/p/7453778.html