第一个简单爬虫代码
import requests
from bs4 import BeautifulSoup
link = "http://www.santostang.com/"
headers = {'User-Agent':"Mozilla/5.0(Windows;U;Windows NT6.1;en-US;rv:1.9.1.6)Gecko/20091201 Firefox/3.5.6"}
r = requests.get(link,headers = headers)
soup = BeautifulSoup(r.text,'lxml')
title = soup.find("h1",class_="post-title").a.text.strip()#strip()函数去除首尾空格
print(title)
with open("title.txt","a+")as f:
f.write(title)
f.close()
截图:路径(C:\Users\asus\title.txt)
import requests
headers = {
'Host': 'www.santostang.com',
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.100 Safari/537.36'
}
r = requests.get("http://www.santostang.com/",headers = headers)*第一参数是url 第二个参数是请求头
print("响应状态码:",r.status_code)
import requests
from bs4 import BeautifulSoup
def get_movies():
headers={
'Host': 'movie.douban.com',
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.100 Safari/537.36'
}
movie = []
for i in range(0,10):
link = "https://movie.douban.com/top250?start=" + str(i*25) + "&filter="
r = requests.get(link,headers = headers,timeout=10)
print(str(i+1),"页响应状态码:",r.status_code)
soup = BeautifulSoup(r.text,"lxml")
list1 = soup.find_all('div',class_="hd")*注意class_不是class
for each in list1:
temp = each.a.span.text.strip()
movie.append(temp)
return movie
movies = get_movies()
print(movies)
``![
![\](https://img-blog.csdnimg.cn/a5ae7a0112ba4b1eb5d51d4bdce6092c.png)
```python
import requests
from bs4 import BeautifulSoup
headers = {
'Host': 'movie.douban.com',
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.100 Safari/537.36'
}
date = []
list = {}
link = 'https://movie.douban.com/top250'
r = requests.get(link,headers = headers)
soup = BeautifulSoup(r.text,"lxml")
for i in range(0,25):
title = soup.find_all('div',class_="item")[i]
t = title.find_all("span")
for j in range(0,len(t)):
date.append(t[j].text)
# print(date)
# print(t[j].text,end=" ")
list[i]=date
date = []
print(list)
```获取电影信息
![在这里插入图片描述](https://img-blog.csdnimg.cn/ed3cf8f9240145f091bd9dfdbe90e598.png)