python爬虫入门

request是python的http访问库，模仿浏览器象服务器发出请求
import requests
from bs4 import BeautifulSoup
response = requests.get('https://ncov.dxy.cn/ncovh5/view/pneumonia')
print(response.text)(容易出现乱码)
text = response.content.decode()/先拿到返回数据的二进制，然后用默认的utf8解码
print(text)

BeautifulSoup库是从html中筛选出有用的数据，可以用id,attr,text筛选
import requests
from bs4 import BeautifulSoup
response = requests.get('https://ncov.dxy.cn/ncovh5/view/pneumonia')
html = response.content.decode()
print(html)
soup = BeautifulSoup(html, 'lxml')
script = soup.find(id="getListByCountryTypeService2true")
print(script)

正则表达式的使用(提取其中有用的json字串)
import requests
from bs4 import BeautifulSoup
import re
response = requests.get('https://ncov.dxy.cn/ncovh5/view/pneumonia')
html = response.content.decode()
soup = BeautifulSoup(html, 'lxml')
script = soup.find(id="getListByCountryTypeService2true")
text = script.string
json_str = re.findall(r'\[.+\]', text)
print(json_str)

json字串转python字串
import json
str = '''[{
    "姓名":"小明",
    "性别":"男",
    "地址":"芜湖"
},
{
    "姓名":"小天",
    "性别":"男",
    "地址":"上海"
}
]'''

rs = json.loads(str)
print(rs)

json文件转python字串
import json
with open('aaa.txt', encoding="utf8") as f:
    rs = json.load(f)
    print(rs)
其中encoding="utf8"很重要,不加会报错`
`

将python字串转换为json字串
import json
with open('aaa.txt', encoding="utf8") as f:
    rs = json.load(f)
json_str = json.dumps(rs, ensure_ascii=False)
print(json_str)

将python字串直接写入json文件(ensure_ascii=False不加是二进制，encoding="utf8"不加是乱码)
import json
with open('aaa.txt', encoding="utf8") as f:
    rs = json.load(f)
json_str = json.dumps(rs, ensure_ascii=False)
print(json_str)

with open('sss.json', 'w', encoding='utf8') as f:
    json.dump(rs, f, ensure_ascii=False)

以post方法访问网页
import urllib.request
import urllib.parse


data = bytes(urllib.parse.urlencode({"hello":"wrold"}), encoding="utf-8")
response = urllib.request.urlopen("http://httpbin.org/post", data=data)
print(response.read().decode("utf-8"))

携带更多信息访问网页
import urllib.request
import urllib.parse
url = "http://httpbin.org/post"
data = bytes(urllib.parse.urlencode({"name":"jack"}), encoding="utf-8")
headers = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/85.0.4183.102 Safari/537.36"}
req = urllib.request.Request(url=url, data=data, headers=headers, method="POST")
response = urllib.request.urlopen(req)
print(response.read().decode())