爬虫定制未完成

最新推荐文章于 2024-08-29 21:24:41 发布

张一爻

最新推荐文章于 2024-08-29 21:24:41 发布

阅读量131

点赞数

分类专栏： python学习

本文链接：https://blog.csdn.net/weixin_43069769/article/details/101752245

版权

python学习专栏收录该内容

54 篇文章 0 订阅

订阅专栏


import urllib
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import time
import os
import sys
import re
import requests
import random

class constant:
    #浏览器识标
    User_Agent = ["Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.95 Safari/537.36",
                  "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/35.0.1916.153 Safari/537.36",
                  "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:30.0) Gecko/20100101 Firefox/30.0",
                  "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_2) AppleWebKit/537.75.14 (KHTML, like Gecko) Version/7.0.3 Safari/537.75.14",
                  "Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.2; Win64; x64; Trident/6.0)",
                  'Mozilla/5.0 (Windows; U; Windows NT 5.1; it; rv:1.8.1.11) Gecko/20071127 Firefox/2.0.0.11',
                  'Opera/9.25 (Windows NT 5.1; U; en)',
                  'Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; .NET CLR 1.1.4322; .NET CLR 2.0.50727)',
                  'Mozilla/5.0 (compatible; Konqueror/3.5; Linux) KHTML/3.5.5 (like Gecko) (Kubuntu)',
                  'Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.8.0.12) Gecko/20070731 Ubuntu/dapper-security Firefox/1.5.0.12',
                  'Lynx/2.8.5rel.1 libwww-FM/2.14 SSL-MM/1.4.1 GNUTLS/1.2.9',
                  "Mozilla/5.0 (X11; Linux i686) AppleWebKit/535.7 (KHTML, like Gecko) Ubuntu/11.04 Chromium/16.0.912.77 Chrome/16.0.912.77 Safari/535.7",
                  "Mozilla/5.0 (X11; Ubuntu; Linux i686; rv:10.0) Gecko/20100101 Firefox/10.0 "]
    #随机选择浏览器识标
    Hearer = User_Agent[random.randint(0, len(User_Agent))]




class functions():
    #获得爬虫的html文件
    def get_html_text(time):
        #网址名称必须是字符串
        url=input()
        return urllib.request.urlopen(str(url), timeout=int(time)).read().decode("utf-8")
    #获得用来分析的html文件
    def get_html_for_analysis(url):
        #网址名称必须是字符串
        return requests.get(str(url)).text

    #创建一个text文档并写入
    def create_an_txt_file(name, body ,file_path):
        # 文件路径，名称，内容必须识字符串
        file = open(file_path + name +time.strftime('%Y-%m-%d', time.localtime())+ '.txt' , 'w') # 也可以创建一个.doc的word文档
        file.write(body)
        print('file name : ' ,file)
        file.close()

    #创建一个csv文件并写入
    def create_an_csv_file(name,body,file_path):
        #文件路径，名称，内容必须识字符串
        return body.pd.to_csv(file_path,mode=a,hearer=False )

    #正则只要两两之间
    def re_function_findall(restart,reend,data):
        return re.findall('r'+restart+'(.*)'+reend,data)

    #正则去除不要的部分
    def re_function_sub(myre,data):
        return re.sub('r'+myre,data)

    #将数值list整成集合
    def change_number_list_to_str(an_list):
        return ''.join([str(i) for i in an_list])




class set_re:

    pass

class run:
    pass

class analysis_and_calculation:
    pass

class savefiles:
    pass

class keybrod_input:
    pass

'''dbug'''