python程序代码保存

实例1 白杨网爬取图片：

// An highlighted block
import requests
import os

url = "http://www.cuc.edu.cn/_upload/article/images/92/67/6e6b9b08471eb4ecfeca0e7091cf/c01c28b9-d48a-43a9-9727-899ba4b6067b.jpg"
root = "D://pics//"#根目录
path = root + url.split('/')[-1]#网络文件名称最后一部分截取
try:
    if not os.path.exists(root):
        os.mkdir(root)#根目录在与否
    if not os.path.exists(path):
        r = requests.get(url)#文件在与否
        with open(path,'wb') as f:
            f.write(r.content)#二进制
            f.close()
            print("文件保存成功")
    else:
        print("文件已存在")
except:
    print("error")

实例2 demo网页爬取文字：

// An highlighted block
import requests
from bs4 import BeautifulSoup
keyword = "python"
try:
     kv = {'wd':keyword}
     demo = requests.get("https://python123.io/ws/demo.html",params=kv)
     print(demo.request.url)#Response对象属性 request
     demo.raise_for_status()#??
     #print(demo.text)
except:
     print("error！")
soup= BeautifulSoup(demo.content,"html.parser")#soup表示解析后的demo页面。
# print(soup.head)
# print(soup.head.contents)
# print(soup.body.contents)#tag类型的和字符串类型的结点都是叶子节点
# print(len(soup.body.contents))
# print(soup.body.contents[1])
# print(soup.title.parent)
# print(soup.html.parent)
#soup = BeautifulSoup("<p>中文</p>,html.parser")
#print(soup.p.string)
print(soup.find_all(string = "Basic Python"))
import re
print(soup.find_all(string = re.compile("python")))

实例3 bs4及requests合用：

// An highlighted block
######为啥不输出？？？？？？？？？？？？？？？？？？？？？
import requests
from bs4 import BeautifulSoup
import bs4
#第一步：搭框架：用的函数，主函数
#第二步：写函数
#
def gethtmltext(url):#获取url信息
    try:
        r = requests.get(url,timeout =30 )
        r.raise_for_status()
        r.encoding = r.apparent_encoding
        return r.text
    except:
        return ""
def filluniverlist(ulist,html):#把html的内容放到ulist里
    soup = BeautifulSoup(html,"html.parser")
    for tr in soup.find('tbody').children:
       if isinstance(tr,bs4.element.Tag):#对类型做判断，如果不是bs4库的tag类型，则过滤掉。
           tds = tr('td')# tr已经都是tag类，将所有的td标签存进一个列表里
           ulist.append([tds[0].string,tds[1].string,tds[2].string])#在列表末尾添加新的对象
            #大学排名 名称 分数

def printlist(ulist,num):#要打印的参数num
    tplt = "{0:^10}\t{1:{3}^10}\t{2:^10}"
    print(tplt.format("排名","学校名称","总分",chr(12288)))
    for i in range(num):#各学校的信息循环打印
        u = ulist[i]
        print(tplt.format(u[0],u[1],u[2],chr(1228)))
def main():
    uninfo = []#大学信息方法到列表中
    url = "https://www.shanghairanking.cn/rankings/bcur/2020"
    html = gethtmltext(url)
    filluniverlist(uninfo,html)
    printlist(uninfo,20)
main()

实例4 爬取《斗破苍穹》小说：

// An highlighted block
import requests
import re
import time
#伪装成浏览器
headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.141 Safari/537.36'}
#
f = open('D:/pics/doupo.txt','+a')
#
def get_info(url):
    res = requests.get(url,headers=headers)#获取网页html
    if res.status_code==200:
        contents = re.findall('<p>(.*?)</p>',res.content.decode('utf-8'),re.S)
        for content in contents:
            f.write(content+'\n')#正则获取数据写入txt文件中
    else:
        pass
if __name__=='__main__':
    urls = ['http://www.doupoxs.com/doupocangqiong/{}.html'.format(str(i))for i in range(2,1665)]#构造多页的url
    for url in urls:
        get_info(url)
        time.sleep(1)
f.close()

实例5 淘宝爬取商品信息：

// An highlighted block
import requests
import re
def gethtmltext(url):
    try:
        r = requests.get(url,timeout = 30)
        r.raise_for_status()
        r.encoding = r.apparent_encoding
        return r.text
    except:
        return ""
def parsepage(ilt,html):#解析网页,ilt结果列表
    try:
        plt = re.findall(r'\"view_price\"\:\"[\d\.]*\"',html)#\是转意字符,价格
        tlt = re.findall(r'\"title\"\:\".*?\"',html)#名称
        for i in range(len(plt)):
            price = eval(plt[i].split(':')[1])#切掉不需要的字符
            title = eval(tlt[i].split(':')[1])#以:为断点,断句 eval是干啥的函数
            ilt.append([price,title])#把提取出来的价格,名称往列表里放.0位置是商品价格,1位置是商品的名称
    except:
        print("")

def printlist(ilt):
    tplt = "{:4}\t{:8}\t{:16}"#\t 缩进4个字符相当于Tab键,:之前是输出的起始位置.python是从0开始输出的:之后是位宽
    print(tplt.format("序号","价格","商品名称"))#表头
    count = 0#序号
    for g in ilt:
        count = count+1
        print(tplt.format(count,g[0],g[1]))
    print("")
def main():#主函数
    goods = '书包'
    depth = 3# 当前页与第二页
    start_url = 'https://www.taobao.com/search?q='+goods
    infolist = []#输出结果
    for i in range(depth):
        try:
            url = start_url+'&s='+str(44*i)
            html = gethtmltext(url)#获取
            parsepage(infolist,html)#解析
        except:
            continue
    printlist(infolist)

main()# 调用整个程序