python 网络爬虫

编程实现：利用requests爬虫库和beautifulsoup4解析库实现指定网站文章标题和对应网址爬取。

已知：

（1）爬取网址：信息学院首页

（2）编程实现爬取"计算机科学系"在"信息学院"主页上的网址

from bs4 import BeautifulSoup
import requests
url="https://coi.hzau.edu.cn/index.htm"
headers={'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) '+\
         'AppleWebkit/537.36 (KHTML, like Gecko)'+\
         'Chrome/99.0.4844.51'+\
         'Safari/537.3'+\
         'Edg/99.0.1150.39'}
res=requests.get(url,headers=headers)
res.encoding='utf-8'
soup=BeautifulSoup(res.text,"lxml")
data=soup.find_all('a')
with open(r'data.txt','w') as f:
    for i in data:
        f.write(i.get('href')+" "+i.text+'\n')
with open(r'data.txt','r') as f:
    a=input("请输入想查找的专业：")
    l=[]
    lines=f.readlines()
    for line in lines:
        if a in line:
            if line not in l:
                print("网址为:","https://coi.hzau.edu.cn/"+line)
            l.append(line)

import requests
from bs4 import BeautifulSoup
url="http://coi.hzau.edu.cn/"
header={"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/113.0.0.0 Safari/537.36 Edg/113.0.1774.50"}
res=requests.get(url,headers=header)
res.encoding='utf-8'
html=res.text
soup=BeautifulSoup(html,'html.parser')
a=soup.find_all('a')
for i in a:
    i=str(i)
    if("计算机科学" in i):
        i=i.split('"')
        url=i[1]
        break
baseurl="http://coi.hzau.edu.cn/"
print("信息学院计算机科学系的主页网址为：",baseurl+url)
url=baseurl+url
response=requests.get(url,headers=header)
response.encoding='utf-8'
soup=BeautifulSoup(response.text,"html.parser")
a=soup.find_all("a")
tn=[]
for i in a:
    href=str(i)
    if "info" in href:
        href=href.split(">")[1]
        tn.append(href.split("<")[-2])
print("计算机科学系的老师有：",' '.join(tn))