urllib requests 常用

最新推荐文章于 2023-12-09 11:36:27 发布

置顶 18923489164

最新推荐文章于 2023-12-09 11:36:27 发布

阅读量365

点赞数

分类专栏：爬虫

本文链接：https://blog.csdn.net/AnYeZhiYin/article/details/105440971

版权

爬虫专栏收录该内容

117 篇文章 15 订阅

订阅专栏


import re
from bs4 import BeautifulSoup as bs
from urllib.request import quote
import pickle

import requests
import urllib.request
browser.set_page_load_timeout(30)
browser.set_script_timeout(30)
import time

# 格式化成2016-03-20 11:45:39形式
print (time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()))

# 格式化成Sat Mar 28 22:24:24 2016形式
print (time.strftime("%a %b %d %H:%M:%S %Y", time.localtime()))
  
# 将格式字符串转换为时间戳
a = "Sat Mar 28 22:24:24 2016"
print (time.mktime(time.strptime(a,"%a %b %d %H:%M:%S %Y")))
def cun():
    
    file = open('','wb')
    pickle.dump(lian,file)
    file.close()
    file = open('','wb')
    pickle.dump(chong,file)
    file.close()   


#字典合并函数
 def dic(dic1,dic2):
    for i in dic1:
        dic2[i]= dic1[i]

    return dic2

try:

    file = open('.pkl','rb')
    lian = pickle.load(file)
    file.close()
except:
    lian =[]
    file = open('.pkl','wb')
    pickle.dump(lian,file)
    file.close()
    file = open('.pkl','rb')
    lian = pickle.load(file)
def nwe_pkl(w):
    data ={}
    file = open(w,'wb')
    pickle.dump(data,file)
    file.close()
    file = open(w,'rb')
    data = pickle.load(file)
    file.close()
    return data

def cun_pkl(w):
    
    file = open(w,'rb')
    data = pickle.load(file)
    file.close()

def lian(url):

    headers ={}

    headers['User-Agent']='Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.132 Safari/537.36'
    
    req = urllib.request.Request(url,headers = headers)

    html = urllib.request.urlopen(req)

    html = html.read()

    html = bs(html,'lxml')

    return html


    #data={}
    #data['']=''

    #data = urllib.parse.urlencode(data).encode('utf-8')
    #html = urllib.request.urlopen(req,data)



def lian2(url):
    
    headers ={}

    headers['User-Agent']='Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.132 Safari/537.36'

    html = requests.get(url,headers=headers)

    #data={}
    #data['']=''

    #html = requests.post(url,headers=headers,data=data)