一、urllib库转换参数、发送请求
from urllib import request,parse params = parse.urlencode(params_dict) # 将字典(或元组——每个元素是包含两个元素的小元组)转换为字符串格式的查询参数 req = request.Request(url,data=params.encode()) # 创建Request对象,封装URL地址和请求参数 注意:data参数必须是字节数据,如果传递data参数,则默认是POST请求。
response = request.urlopen(req) # urlopen()可以接收URL字符串或者Request对象
1.1 百度翻译爬取
代码演示
from urllib import request,parse url = 'https://fanyi.baidu.com/sug' # 百度翻译接口 params_dict = { 'kw':'banana', } params = parse.urlencode(params_dict) # 将字典(或元组——每个元素是包含两个元素的小元组)转换为字符串格式的查询参数 print("转换后的查询字符串参数:"+params) req = request.Request(url,data=params.encode()) #data不传默认为get请求,传参默认为post请求 response = request.urlopen(req) if response.status == 200: print("恭喜,请求成功!") print(response.read().decode()) print("***********以下是requests库的解法*************") import requests post_response = requests.post(url,data=params_dict) if post_response.status_code == 200: print("requests库post请求方式成功~~~") print(post_response.text)
from urllib import request,parse url = 'https://fanyi.baidu.com/v2transapi' params_dict = { 'from': 'en', 'to': 'zh', 'query': 'banana', 'transtype': 'realtime', 'simple_means_flag': '3', 'sign': '594751.913422', 'token': '4d0a1a6ebe0a184c8c44bb5d3477330b', } headers = { 'Host': 'fanyi.baidu.com', 'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.77 Safari/537.36', 'Cookie': 'BIDUPSID=8CDE36310DA13D77BA6F2F1EFCCDAD60; PSTM=1539593553; BAIDUID=8CDE36310DA13D77BA6F2F1EFCCDAD60:SL=0:NR=10:FG=1; BDORZ=B490B5EBF6F3CD402E515D22BCDA1598; REALTIME_TRANS_SWITCH=1; FANYI_WORD_SWITCH=1; HISTORY_SWITCH=1; SOUND_SPD_SWITCH=1; SOUND_PREFER_SWITCH=1; H_PS_PSSID=26522_1424_21094_27377; Hm_lvt_64ecd82404c51e03dc91cb9e8c025574=1540992599,1541036898; Hm_lpvt_64ecd82404c51e03dc91cb9e8c025574=1541041590; delPer=0; PSINO=1; locale=zh; to_lang_often=%5B%7B%22value%22%3A%22en%22%2C%22text%22%3A%22%u82F1%u8BED%22%7D%2C%7B%22value%22%3A%22zh%22%2C%22text%22%3A%22%u4E2D%u6587%22%7D%5D; from_lang_often=%5B%7B%22value%22%3A%22zh%22%2C%22text%22%3A%22%u4E2D%u6587%22%7D%2C%7B%22value%22%3A%22en%22%2C%22text%22%3A%22%u82F1%u8BED%22%7D%5D' } params = parse.urlencode(params_dict) req = request.Request(url,data=params.encode(),headers=headers) response = request.urlopen(req) if response.status == 200: print("请求成功!") print(response.read().decode())
二、替换原URL中含有%的部分
from urllib import request real_url = request.unquote("含有%的url") # 将url中的%替换为原本的字符
2.1 豆瓣电影爬取
代码演示
动态抓包
from urllib import request,parse # raw_url = 'https://movie.douban.com/j/chart/top_list' # real_url = request.unquote(raw_url) # 将url中的%替换为原本的字符 def spider_movie_rating(category,start,limit=20): params_dict = { 'type':category, 'interval_id':'100:90', 'action':'', 'start':start, 'limit':limit } params = parse.urlencode(params_dict) # 将字典转换为查询字符串 url = 'https://movie.douban.com/j/chart/top_list' url += "?"+params req = request.Request(url) response = request.urlopen(req) m = (response.read().decode('utf-8')) if response.status == 200: with open('movie_rating_%d.txt'%start,'wb') as f: f.write(m.encode()) print("存储成功") if __name__ == '__main__': for i in range(5): spider_movie_rating(23,20*i)
爬取每条电影中对应的图片
import json from urllib import request with open('movie_rating_0.txt','r') as f: movie_list = json.load(f) for movie in movie_list: print() imgpath = movie['cover_url'] request.urlretrieve(imgpath,'./images/'+imgpath.split('/')[-1]) print(imgpath.split('/')[-1]+"保存成功!")
三、BeautifulSoup的其他用法:
1. 可以与正则表达式联合使用 import re soup = BeautifulSoup(要解析的文本,'lxml') soup.find_all(re.compile("正则表达式"))
可以通过CSS选择器获取标签soup.select("CSS选择器")
3.1 搜狐首页爬取获取制定标签内容
代码演示
from bs4 import BeautifulSoup import requests import re # url = 'http://www.sohu.com' # r = requests.get(url) # with open('sohu.html','wb') as f: # f.write(r.text.encode()) #注释部分为爬取网页部分 #BeautifulSoup的使用 with open('sohu.html','rb') as f: html = f.read().decode() soup = BeautifulSoup(html,'lxml') for tag in soup.find_all(re.compile("^m")): # 获取以m开头的标签 print(tag) print(soup.select("div ul")) # 获取所有div下的ul标签,以列表形式返回 print(soup.select("div#sohuTopc > div.ph-link"))
四、自定义Django项目实现Ajax传参
代码演示
views
from django.http import JsonResponse from django.shortcuts import render def show_fruits(request): fruits = ["苹果","香蕉","梨","橘子"] return render(request,'fruits.html',locals()) def show_tasted_fruits(request): tasted_fruits = ["好吃的火龙果","好吃的菠萝","可口的猕猴桃"] return JsonResponse({'tasted_fruits':tasted_fruits}) #必须传字典
templates
<!DOCTYPE html> {% load static %} <html lang="en"> <head> <meta charset="UTF-8"> <title>水果信息</title> <script type="text/javascript" src="{% static 'js/jquery-3.1.1.js' %}"></script> <script type="text/javascript" src="{% static 'js/fruit.js' %}"></script> </head> <body> <ul> {% for fruit in fruits %} <li> {{ fruit }} </li> {% endfor %} </ul> <div id="tasted" style="color:red"> </div> </body> </html> <!----------------------------------------------------------------------------------> <!js/fruit.js> $(function(){ $.ajax({ url:'/myapp/tasted/', type:'GET', success:function(data){ //回调函数 var tasted_fruits = data["tasted_fruits"]; for(var key in tasted_fruits){ $("div#tasted").append(tasted_fruits[key]+"<br/>"); } alert('success'); } }) })
urls
#总路由 from django.contrib import admin from django.urls import path,include urlpatterns = [ path('admin/', admin.site.urls), path('myapp/',include('myapp.urls')), ] #子路由 from django.urls import path from myapp.views import * urlpatterns = [ path('fruits/',show_fruits), path('tasted/',show_tasted_fruits) ]
五、homework
5.1 利用天气API 爬取天气情况并保存
from urllib import request import json url = 'http://www.weather.com.cn/data/sk/101110101.html' url1 = "http://www.weather.com.cn/data/sk/101110103.html" req = request.Request(url1) response = request.urlopen(req) if response.status == 200: weather_str = response.read().decode() weather_dict = json.loads(weather_str) print(weather_dict) with open('weather.txt','w',encoding='utf-8') as f: info_dict = weather_dict["weatherinfo"] city = info_dict["city"] temperature = info_dict["temp"] wind_direction = info_dict["WD"] f.write("城市:"+city+"\n") f.write("温度:" + temperature+"\n") f.write("风向:" + wind_direction+"\n") print("天气信息保存成功!")