python 搜索引擎 实验楼的源码_Python爬虫，爬取实验楼全部课程

1 #引入所需要的库

2 importtime3 importrequests4 from peewee import *

5 from lxml importetree6 #这个程序要倒着看

7

8 #这个是连接数据库的，名字和密码根据自己情况修改

9 db = MySQLDatabase('shiyanlou', user='root', passwd='xxxxxx')10

11

12 classCourse(Model):13 title =CharField()14 teacher =CharField()15 teacher_courses =IntegerField()16 tag =CharField()17 study_num =IntegerField()18 content =CharField()19

20 classMeta:21 database =db22

23

24 Course.create_table()25

26

27 defparse_content(url, title, tag, study_num):28 print('课程地址：' +url)29 res =requests.get(url)30 xml =etree.HTML(res.text)31 #获取页面里的简介

32 try:33 content = xml.xpath('//meta[@name="description"]/@content')[0]34 exceptException as e:35 content = '无'

36 #获取老师名字

37 try:38 teacher =xml.xpath(39 '//div[@class="sidebox mooc-teacher"]//div[@class="mooc-info"]/div[@class="name"]/strong/text()')[0]40 exceptException as e:41 teacher = '匿名'

42 #获取老师发表课程数目

43 try:44 teacher_courses =xml.xpath(45 '//div[@class="sidebox mooc-teacher"]//div[@class="mooc-info"]/div[@class="courses"]/strong/text()')[0]46 exceptException as e:47 teacher_courses = '未知'

48 #存入数据库

49 try:50 course = Course(title=title, teacher=teacher,51 teacher_courses=int(teacher_courses), tag=tag, study_num=int(study_num), content=content)52 course.save()53 exceptException as e:54 print('一条数据存取失败')55

56

57 defget_course_link(url):58 #获取每一页的信息，传给下一个函数

59 response =requests.get(url)60 xml =etree.HTML(response.text)61 #contains()是包含的意思

62 courses =xml.xpath(63 '//div[contains(@class, "col-md-3") and contains(@class, "col-sm-6") and contains(@class, "course")]')64 for course incourses:65 try:66 url = 'https://www.shiyanlou.com' + course.xpath('.//a/@href')[0]67 exceptException as e:68 print('一个课程页面未获得')69 continue

70 title = course.xpath('.//div[@class="course-name"]/text()')[0]71 study_people =course.xpath(72 './/span[@class="course-per-num pull-left"]/text()')[1].strip()73 #study_people = int(study_people)

74 try:75 tag =course.xpath(76 './/span[@class="course-money pull-right"]/text()')[0]77 exceptException as e:78 tag = "普通"

79 parse_content(url=url, title=title, tag=tag, study_num=study_people)80 #time.sleep(0.5)

81

82

83 defmain():84 #通过requests库的get获得目标地址的返回信息，类型为Response

85 response = requests.get('https://www.shiyanlou.com/courses/')86 #将返回信息的文本转化为xml树，可以通过xpath来进行查询

87 xml =etree.HTML(response.text)88 #由分析网页源代码可以总结，url分页模式，只有最后的数字不一样

89 course_link = 'https://www.shiyanlou.com/courses/?category=all&course_type=all&fee=all&tag=all&page={}'

90 #这里获得最大页数就可以了,xpath()函数里的便是寻找路径了

91 #//会在全文来进行查找，//ul则是查找全文的ul标签，//ul[@class="pagination"]会仅查找有class属性，

92 #且为"pagination"的标签，之后/li是查找当前的ul标签下的li标签(仅取一层),取查询到的列表倒数第二个标签

93 #为li[last()-1],/a/text()查询a标签里的文本内容

94 page = xml.xpath('//ul[@class="pagination"]/li[last()-1]/a/text()')95 if len(page) != 1:96 print('爬取最大页数时发生错误！！')97 returnNone98 #page原是一个列表，这里取出它的元素，并转化为Int型

99 page =int(page[0])100 #将每一页的url传给get_course_link函数进行处理

101 for i in range(1, page + 1):102 #填入course_link,获取完整url

103 url =course_link.format(i)104 print('页面地址：' +url)105 #调用另一个函数

106 get_course_link(url)107

108

109 if __name__ == '__main__':110 #调用main函数

111 main()112

113 #[Finished in 218.5s]