获取url

import requests
from bs4 import BeautifulSoup
import re


class Get_links(object):
    def __init__(self):
        self.url = 'http://www.jianfeiketang.com'
        self.headers = {
            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3724.8 Safari/537.36",
        }

    def parse(self):
        response = requests.get(url=self.url, headers=self.headers)
        html = response.content.decode()

        links = []
        soup = BeautifulSoup(html, "html.parser")
        url_list = soup.find_all('a')
        for link in url_list:
            links.append(link.get('href'))
        return links

    def save(self, links):
        for link in links:
            with open('../result/jf.txt', 'a', encoding='utf-8') as f:
                f.write(link + '\n')
        print('保存成功...')

    def guolv(self):
        with open('../result/jf.txt', 'r', encoding='utf-8') as f:
            lines = f.readlines()
            for line in lines:  # 循环处理每行数据
                urls = re.findall(r"http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\(\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+",
                                  line)
                for url in urls:
                    with open('./jfresult.txt', 'a', encoding='utf-8') as f:
                        f.write(url + '\n')
        print('清洗完毕。。。')


if __name__ == '__main__':
    get_links = Get_links()
    links = get_links.parse()
    get_links.save(links)
    get_links.guolv()

转载于:https://www.cnblogs.com/victorstudy/p/11425895.html