python爬虫-绝对领域

coding:utf-8

import re
import urllib.request
import requests
import time
from time import sleep
from bs4 import BeautifulSoup
import threading
import os

获取网页源码

def gethtml(Waite_url):
papg = urllib.request.urlopen(Waite_url) #打开图片的网址
html = papg.read() #用read方法读成网页源代码，格式为字节对象
html = html.decode(‘UTF-8’) #定义编码格式解码字符串(字节转换为字符串)
return html

def get_page(url,page_num): #获取分页网页的代码
pageList =[] #建立列表保存网页的原代码
for i in range(1,page_num +1):
formdata ={‘type’:’index’ ,
‘paged’: i}
try:
r =requests.post(url,data =formdata) #发起网页请求
r.raise_for_status()
r.encoding = r.apparent_encoding
pageList.append(r.text)
print(‘链接成功’)
except:
print(‘链接失败’)
return pageList
def mylist(listname): #创建文件夹
#判断文件夹名是否存在不能建立文件名的特殊字符
try:
listname=re.sub(‘[\/:*?”<>|]’,”_”,listname)
except:
pass
path=”F:\juedui\”+listname
if os.path.exists(path): # 判断是否存在同名文件夹
path = path + str(1)
else:
pass
os.makedirs(path)
return path

获取当前毫秒级日期格式时间

def get_time_stamp(ct):
local_time = time.localtime(ct)
data_head = time.strftime(“%Y-%m-%d %H:%M:%S”, local_time) #当前时间的日期格式
data_secs = (ct - int(ct)) * 1000 #时间毫秒部分
time_stamp = “%s.%03d” % (data_head, data_secs)
stamp = (“”.join(time_stamp.split()[0].split(“-“)) + “”.join(time_stamp.split()[1].split(“:”))).replace(‘.’, ”)
return stamp

def getimg(html):
soup=BeautifulSoup(html,”html.parser”) #获取主页的所有代码
imgre = soup.find_all(target=”_blank”,rel=”bookmark”)
for imgurl in imgre: #循环图片字符串列表并输出 图片列表链接集合
print(imgurl)
name=imgurl.get_text() #获取主 # path=mylist(name)题标题名字
path=mylist(name)
imgurl=imgurl[“href”]
print(imgurl)
newhtml=gethtml(imgurl) #获取子网页的所有代码
newsoup=BeautifulSoup(newhtml,”html.parser”)
newimgre=newsoup.find_all(“div”,attrs={“id”:”content-innerText”}) #刷选出图片链接集合
newimgre=newimgre[0].find_all(“img”)
for newurl in newimgre:
newurl=newurl[“src”]
print(newurl)
t= time.time() #获取系统当前毫秒级时间
nowtime=get_time_stamp(t) #毫秒级转日期格式
urllib.request.urlretrieve(newurl,path + “\” + nowtime + “.jpg”) #命名图片名称并保存在本地

#调用函数
html = get_page(“http://www.jdlingyu.mobi/“,41)
j=0
for i in html:
getimg(i)
j+=1
print(“第%s页输出成功！”%j)