爬虫入门（1）

最新推荐文章于 2024-09-11 08:51:43 发布

linchaoa1989

最新推荐文章于 2024-09-11 08:51:43 发布

阅读量100

点赞数

分类专栏：爬虫文章标签： python 正则表达式

本文链接：https://blog.csdn.net/linchaoa1989/article/details/82956864

版权

爬虫专栏收录该内容

0 篇文章 0 订阅

订阅专栏

# -*- coding: utf-8 -*-
from urllib import request
import re


def re_geturl(mytext, index):
    p1 = r"http.*D" + str(index)  # 正则表达式规则
    http_list = re.findall(p1, mytext)
    http_list = list(map(lambda x: x.replace("amp;", ""), http_list))
    http_list = list(set(http_list))  # 去重
    for url in http_list:
        print(url)


def get_prestige_hd():
    """
    读取网页中的url
    :return: url_list
    """
    url = "http://fxxxx.com/forum.php?mod=forumdisplay&fid=131&page="
    for i in range(1, 10):
        cur_url = url + str(i)
        print(cur_url)
        response = request.urlopen(cur_url)  # 打开连接
        html = response.read()
        html = html.decode("utf-8")
        re_geturl(html, i)
        break
    return ""


if __name__ == "__main__":
    get_prestige_hd()