收藏关注不迷路！！
🌟文末获取源码+数据库🌟
感兴趣的可以先收藏起来，还有大家在毕设选题（免费咨询指导选题），项目以及论文编写等相关问题都可以给我留言咨询，希望帮助更多的人
前言
💗博主介绍：✨全网拥有20W+粉丝、CSDN作者、博客专家、全栈领域优质创作者、平台优质Java创作者、专注于Java、小程序、python、安卓技术领域和毕业项目实战✌💟！✨💗
👇🏻 精彩专栏 推荐订阅👇🏻
计算机毕业设计设计精品实战案例✅
感兴趣的可以先收藏起来，还有大家在毕设选题（免费咨询指导选题），项目以及论文编写等相关问题都可以给我留言咨询，博主免费解答、希望可以帮助更多人
详细视频演示
文章底部名片，获取项目的完整演示视频，免费解答技术疑问
项目介绍
当今社会进入了科技进步、大数据快速发展的新时代。大数据的高校电子图书馆也不断加强，计算机技术对经济社会发展和人民生活改善的影响也日益突出，人类的生存和思考方式也产生了变化。传统的高校电子图书馆采取了人工的管理方法，但这种管理方法存在着许多弊端，比如效率低下、安全性低以及信息传输的不准确等，同时由于高校电子图书馆管理中中会形成众多的图书馆书籍数据，比如书籍类型、出版社图书、图书量情况、图书馆书籍、图书馆书籍总数/图书借阅总数、用户借阅、借阅状态等，更改和维护等较为麻烦的管理问题，同时由于当下人民群众对高校电子图书馆管理的需求也日益高涨，各高校电子图书馆管理开展全新的改革以满足时代的需求。根据此问题，研发一套高校电子图书馆的大数据平台，既能够大大提高图书馆书籍信息的检索、变更与维护的工作效率，也能够方便信息系统的管理运用，从而减少信息管理成本，提高效率。
该高校电子图书馆的大数据平台采用B/S架构、开发语言使用python，并采用Hadoop技术以及 springboot框架进行开发。本系统主要设计并完图书馆书籍、图书借阅、图书归还、书籍入库等功能，进行维护与管理。该系统操作简单，界面设计简单，不仅能基本满足目前高校电子图书馆的日常管理工作，而且能有效降低人员成本和时间成本，为高校电子图书馆的大数据平台工作提供方便。
技术介绍
开发语言：Java
框架：springboot
JDK版本：JDK1.8
服务器：tomcat7
数据库：mysql
数据库工具：Navicat11
开发软件：eclipse/myeclipse/idea
Maven包：Maven
功能介绍
管理员端的功能主要是开放给系统的管理人员使用，能够对用户的信息进行管理，包括对用户管理、图书馆书籍管理、图书借阅管理、图书归还管理、书籍入库管理进行查看，修改和删除、新增等。
对本系统进行全面的系统功能的分析，可以得出高校电子图书馆的大数据平台的功能模块图，如图4-1所示。
图4-1 系统功能模块图
核心代码
# # -*- coding: utf-8 -*-
# 数据爬取文件
import scrapy
import pymysql
import pymssql
from ..items import DoubanItem
import time
from datetime import datetime,timedelta
import datetime as formattime
import re
import random
import platform
import json
import os
import urllib
from urllib.parse import urlparse
import requests
import emoji
import numpy as np
import pandas as pd
from sqlalchemy import create_engine
from selenium.webdriver import ChromeOptions, ActionChains
from scrapy.http import TextResponse
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.support.wait import WebDriverWait
# 豆瓣高分
class DoubanSpider(scrapy.Spider):
name = 'doubanSpider'
spiderUrl = 'https://read.douban.com/j/kind/'
start_urls = spiderUrl.split(";")
protocol = ''
hostname = ''
realtime = False
def __init__(self,realtime=False,*args, **kwargs):
super().__init__(*args, **kwargs)
self.realtime = realtime=='true'
def start_requests(self):
plat = platform.system().lower()
if not self.realtime and (plat == 'linux' or plat == 'windows'):
connect = self.db_connect()
cursor = connect.cursor()
if self.table_exists(cursor, 'l908648a_douban') == 1:
cursor.close()
connect.close()
self.temp_data()
return
data_json={"sort": "book_rating","page": 1,"kind": 100,"query": "\n query getFilterWorksList($works_ids: [ID!]) {\n worksList(worksIds: $works_ids) {\n \n id\n isOrigin\n isEssay\n \n title\n cover(useSmall: false)\n url\n isBundle\n coverLabel(preferVip: true)\n \n \n url\n title\n\n \n author {\n name\n url\n }\n origAuthor {\n name\n url\n }\n translator {\n name\n url\n }\n\n \n abstract\n authorHighlight\n editorHighlight\n\n \n isOrigin\n kinds {\n \n name @skip(if: true)\n shortName @include(if: true)\n id\n \n }\n ... on WorksBase @include(if: true) {\n wordCount\n wordCountUnit\n }\n ... on WorksBase @include(if: false) {\n inLibraryCount\n }\n ... on WorksBase @include(if: false) {\n \n isEssay\n \n ... on EssayWorks {\n favorCount\n }\n \n \n \n averageRating\n ratingCount\n url\n isColumn\n isFinished\n \n \n \n }\n ... on EbookWorks @include(if: true) {\n \n ... on EbookWorks {\n book {\n url\n averageRating\n ratingCount\n }\n }\n \n }\n ... on WorksBase @include(if: false) {\n isColumn\n isEssay\n onSaleTime\n ... on ColumnWorks {\n updateTime\n }\n }\n ... on WorksBase @include(if: true) {\n isColumn\n ... on ColumnWorks {\n isFinished\n }\n }\n ... on EssayWorks {\n essayActivityData {\n \n title\n uri\n tag {\n name\n color\n background\n icon2x\n icon3x\n iconSize {\n height\n }\n iconPosition {\n x y\n }\n }\n \n }\n }\n highlightTags {\n name\n }\n ... on WorksBase @include(if: false) {\n fanfiction {\n tags {\n id\n name\n url\n }\n }\n }\n \n \n ... on WorksBase {\n copyrightInfo {\n newlyAdapted\n newlyPublished\n adaptedName\n publishedName\n }\n }\n\n isInLibrary\n ... on WorksBase @include(if: false) {\n \n fixedPrice\n salesPrice\n isRebate\n \n }\n ... on EbookWorks {\n \n fixedPrice\n salesPrice\n isRebate\n \n }\n ... on WorksBase @include(if: true) {\n ... on EbookWorks {\n id\n isPurchased\n isInWishlist\n }\n }\n ... on WorksBase @include(if: false) {\n fanfiction {\n fandoms {\n title\n url\n }\n }\n }\n ... on WorksBase @include(if: false) {\n fanfiction {\n kudoCount\n }\n }\n \n }\n }\n ","variables": {}}
pageNum = 1 + 1
for page in range(1, pageNum):
data_json["page"] = page
raw = json.dumps(data_json)
yield scrapy.FormRequest(
url=self.spiderUrl,
body=raw,
method='POST',
callback=self.parse,
headers={'Content-Type': 'application/json'},
dont_filter=True
)
# 列表解析
def parse(self, response):
_url = urlparse(self.spiderUrl)
self.protocol = _url.scheme
self.hostname = _url.netloc
plat = platform.system().lower()
if not self.realtime and (plat == 'linux' or plat == 'windows'):
connect = self.db_connect()
cursor = connect.cursor()
if self.table_exists(cursor, 'l908648a_douban') == 1:
cursor.close()
connect.close()
self.temp_data()
return
print("response:",response.text)
data = json.loads(response.text)
try:
list = data["list"]
except:
pass
for item in list:
fields = DoubanItem()
fields["bookname"] = item["title"]
fields["cover"] = item["cover"]
fields["laiyuan"] = "https://read.douban.com"+item["url"]
fields["wordcount"] = int(item["wordCount"])
fields["salesprice"] = float(item["salesPrice"])
yield scrapy.Request(
url="https://read.douban.com/j"+item["url"],
meta={"fields":fields},
callback=self.detail_parse,
dont_filter=True
)
# 详情解析
def detail_parse(self, response):
print("detail_parse:",response.text)
fields = response.meta['fields']
data = json.loads(response.text)
try:
fields["author"] = data["author"]
except:
fields["author"] = data["original_author"]
try:
fields["chuban"] = data["publisher"]
except:
pass
try:
fields["tags"] = ",".join(i["tag"] for i in data["tags"])
except:
pass
try:
fields["mulu"] = ";".join(i["title"] for i in data["table_of_contents"])
except:
pass
try:
fields["rating"] = data["book_average_rating"]
except:
pass
return fields
# 数据清洗
def pandas_filter(self):
engine = create_engine('mysql+pymysql://root:123456@localhost/spiderl908648a?charset=UTF8MB4')
df = pd.read_sql('select * from douban limit 50', con = engine)
# 重复数据过滤
df.duplicated()
df.drop_duplicates()
#空数据过滤
df.isnull()
df.dropna()
# 填充空数据
df.fillna(value = '暂无')
# 异常值过滤
# 滤出 大于800 和 小于 100 的
a = np.random.randint(0, 1000, size = 200)
cond = (a<=800) & (a>=100)
a[cond]
# 过滤正态分布的异常值
b = np.random.randn(100000)
# 3σ过滤异常值，σ即是标准差
cond = np.abs(b) > 3 * 1
b[cond]
# 正态分布数据
df2 = pd.DataFrame(data = np.random.randn(10000,3))
# 3σ过滤异常值，σ即是标准差
cond = (df2 > 3*df2.std()).any(axis = 1)
# 不满⾜条件的⾏索引
index = df2[cond].index
# 根据⾏索引，进⾏数据删除
df2.drop(labels=index,axis = 0)
# 去除多余html标签
def remove_html(self, html):
if html == None:
return ''
pattern = re.compile(r'<[^>]+>', re.S)
return pattern.sub('', html).strip()
# 数据库连接
def db_connect(self):
type = self.settings.get('TYPE', 'mysql')
host = self.settings.get('HOST', 'localhost')
port = int(self.settings.get('PORT', 3306))
user = self.settings.get('USER', 'root')
password = self.settings.get('PASSWORD', '123456')
try:
database = self.databaseName
except:
database = self.settings.get('DATABASE', '')
if type == 'mysql':
connect = pymysql.connect(host=host, port=port, db=database, user=user, passwd=password, charset='utf8')
else:
connect = pymssql.connect(host=host, user=user, password=password, database=database)
return connect
# 断表是否存在
def table_exists(self, cursor, table_name):
cursor.execute("show tables;")
tables = [cursor.fetchall()]
table_list = re.findall('(\'.*?\')',str(tables))
table_list = [re.sub("'",'',each) for each in table_list]
if table_name in table_list:
return 1
else:
return 0
# 数据缓存源
def temp_data(self):
connect = self.db_connect()
cursor = connect.cursor()
sql = '''
insert into `douban`(
id
,bookname
,author
,cover
,laiyuan
,wordcount
,salesprice
,chuban
,tags
,mulu
,rating
)
select
id
,bookname
,author
,cover
,laiyuan
,wordcount
,salesprice
,chuban
,tags
,mulu
,rating
from `l908648a_douban`
where(not exists (select
id
,bookname
,author
,cover
,laiyuan
,wordcount
,salesprice
,chuban
,tags
,mulu
,rating
from `douban` where
`douban`.id=`l908648a_douban`.id
))
'''
cursor.execute(sql)
connect.commit()
connect.close()
数据库参考
--
-- Current Database: `springbootuxmu8kmc`
--
/*!40000 DROP DATABASE IF EXISTS `springbootuxmu8kmc`*/;
CREATE DATABASE /*!32312 IF NOT EXISTS*/ `springbootuxmu8kmc` /*!40100 DEFAULT CHARACTER SET utf8mb4 */;
USE `springbootuxmu8kmc`;
--
-- Table structure for table `config`
--
DROP TABLE IF EXISTS `config`;
/*!40101 SET @saved_cs_client = @@character_set_client */;
/*!40101 SET character_set_client = utf8 */;
CREATE TABLE `config` (
`id` bigint(20) NOT NULL AUTO_INCREMENT COMMENT '主键',
`name` varchar(100) NOT NULL COMMENT '配置参数名称',
`value` varchar(100) DEFAULT NULL COMMENT '配置参数值',
`url` varchar(500) DEFAULT NULL COMMENT 'url',
PRIMARY KEY (`id`)
) ENGINE=InnoDB AUTO_INCREMENT=9 DEFAULT CHARSET=utf8 COMMENT='配置文件';
/*!40101 SET character_set_client = @saved_cs_client */;
--
-- Dumping data for table `config`
--
LOCK TABLES `config` WRITE;
/*!40000 ALTER TABLE `config` DISABLE KEYS */;
INSERT INTO `config` VALUES (1,'picture1','upload/picture1.jpg',NULL),(2,'picture2','upload/picture2.jpg','jingjishuDetail?id=33'),(3,'picture3','upload/picture3.jpg','https://www.taobao.com'),(7,'APIKey','ihML0NfEM0jokUAuBD9tbYtW',NULL),(8,'SecretKey','WK2bRUFf2uNe3oEoQXULpi8Y37aCvFdD',NULL);
/*!40000 ALTER TABLE `config` ENABLE KEYS */;
UNLOCK TABLES;
--
-- Table structure for table `dgn8vw92_jingjishu`
--
DROP TABLE IF EXISTS `dgn8vw92_jingjishu`;
/*!40101 SET @saved_cs_client = @@character_set_client */;
/*!40101 SET character_set_client = utf8 */;
CREATE TABLE `dgn8vw92_jingjishu` (
`id` bigint(20) NOT NULL DEFAULT '0' COMMENT '主键',
`addtime` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间',
`bookname` varchar(200) CHARACTER SET utf8 DEFAULT NULL COMMENT '书名',
`booktype` varchar(200) CHARACTER SET utf8 DEFAULT NULL COMMENT '文献类型',
`author` varchar(200) CHARACTER SET utf8 DEFAULT NULL COMMENT '著者',
`chubanyear` varchar(200) CHARACTER SET utf8 DEFAULT NULL COMMENT '出版年份',
`chubanshe` varchar(200) CHARACTER SET utf8 DEFAULT NULL COMMENT '出版社',
`sourceku` varchar(200) CHARACTER SET utf8 DEFAULT NULL COMMENT '数源数据库',
`cover` longtext CHARACTER SET utf8 COMMENT '封面',
`laiyuan` longtext CHARACTER SET utf8 COMMENT '来源',
`info` longtext CHARACTER SET utf8 COMMENT '详细信息'
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
/*!40101 SET character_set_client = @saved_cs_client */;
LOCK TABLES `storeup` WRITE;
/*!40000 ALTER TABLE `storeup` DISABLE KEYS */;
INSERT INTO `storeup` VALUES (1,'2023-11-28 02:10:00',1701137386593,34,'jingjishu','书名4','upload/jingjishu_cover4.jpg','1',NULL,NULL),(2,'2023-11-28 02:15:50',1701137386593,38,'jingjishu','书名8','upload/jingjishu_cover8.jpg','1',NULL,NULL);
/*!40000 ALTER TABLE `storeup` ENABLE KEYS */;
UNLOCK TABLES;
--
-- Table structure for table `systemintro`
--
DROP TABLE IF EXISTS `systemintro`;
/*!40101 SET @saved_cs_client = @@character_set_client */;
/*!40101 SET character_set_client = utf8 */;
CREATE TABLE `systemintro` (
`id` bigint(20) NOT NULL AUTO_INCREMENT COMMENT '主键',
`addtime` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间',
`title` varchar(200) NOT NULL COMMENT '标题',
`subtitle` varchar(200) DEFAULT NULL COMMENT '副标题',
`content` longtext NOT NULL COMMENT '内容',
`picture1` longtext COMMENT '图片1',
`picture2` longtext COMMENT '图片2',
`picture3` longtext COMMENT '图片3',
PRIMARY KEY (`id`)
) ENGINE=InnoDB AUTO_INCREMENT=2 DEFAULT CHARSET=utf8 COMMENT='关于我们';
/*!40101 SET character_set_client = @saved_cs_client */;
--
-- Dumping data for table `systemintro`
--
LOCK TABLES `systemintro` WRITE;
/*!40000 ALTER TABLE `systemintro` DISABLE KEYS */;
INSERT INTO `systemintro` VALUES (1,'2023-11-28 01:57:58','系统简介','SYSTEM INTRODUCTION','<p>当遇到挫折或失败，你是看见失败还是看见机会?挫折是我们每个人成长的必经之路，它不是你想有就有，想没有就没有的。有句名言说的好，如果你想一生摆脱苦难，你就得是神或者是死尸。这句话形象地说明了挫折是伴随着人生的，是谁都逃不掉的。人生在世，从古到今，不分天子平民，机遇虽有不同，但总不免有身陷困境或遭遇难题之处，这时候唯有通权达变，才能使人转危为安，甚至反败为胜。大部分的人，一生当中，最痛苦的经验是失去所爱的人，其次是丢掉一份工作。其实，经得起考验的人，就算是被开除也不会惊慌，要学会面对。测试</p>','upload/1701137675098.jpg','upload/1701137678329.jpg','upload/1701137681244.jpg');
/*!40000 ALTER TABLE `systemintro` ENABLE KEYS */;
UNLOCK TABLES;
--
-- Table structure for table `token`
--
DROP TABLE IF EXISTS `token`;
/*!40101 SET @saved_cs_client = @@character_set_client */;
/*!40101 SET character_set_client = utf8 */;
CREATE TABLE `token` (
`id` bigint(20) NOT NULL AUTO_INCREMENT COMMENT '主键',
`userid` bigint(20) NOT NULL COMMENT '用户id',
`username` varchar(100) NOT NULL COMMENT '用户名',
`tablename` varchar(100) DEFAULT NULL COMMENT '表名',
`role` varchar(100) DEFAULT NULL COMMENT '角色',
`token` varchar(200) NOT NULL COMMENT '密码',
`addtime` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '新增时间',
`expiratedtime` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '过期时间',
PRIMARY KEY (`id`)
) ENGINE=InnoDB AUTO_INCREMENT=4 DEFAULT CHARSET=utf8 COMMENT='token表';
/*!40101 SET character_set_client = @saved_cs_client */;
--
-- Dumping data for table `token`
--
LOCK TABLES `token` WRITE;
/*!40000 ALTER TABLE `token` DISABLE KEYS */;
INSERT INTO `token` VALUES (1,11,'账号1','yonghu','用户','ejs1mkd8uc12kxuarr14jrt7k5mnmgyf','2023-11-28 02:08:38','2023-11-28 03:08:38'),(2,1701137386593,'11','yonghu','用户','6timzuf4tpqvh1372bk8isrhnef615a5','2023-11-28 02:09:49','2023-11-28 03:17:42'),(3,1,'admin','users','管理员','ji9a5a2fno6d5gv38jq5okee8ohzvmir','2023-11-28 02:11:10','2023-11-28 03:18:38');
/*!40000 ALTER TABLE `token` ENABLE KEYS */;
UNLOCK TABLES;
--
-- Table structure for table `tushuguihai`
--
DROP TABLE IF EXISTS `tushuguihai`;
/*!40101 SET @saved_cs_client = @@character_set_client */;
/*!40101 SET character_set_client = utf8 */;
CREATE TABLE `tushuguihai` (
`id` bigint(20) NOT NULL AUTO_INCREMENT COMMENT '主键',
`addtime` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间',
`jieyuehao` varchar(200) DEFAULT NULL COMMENT '借阅号',
`bookname` varchar(200) NOT NULL COMMENT '书名',
`shujileixing` varchar(200) NOT NULL COMMENT '书籍类型',
`chubanshe` varchar(200) DEFAULT NULL COMMENT '出版社',
`chubanyear` varchar(200) DEFAULT NULL COMMENT '出版年份',
`cover` longtext COMMENT '封面',
`shuliang` int(11) DEFAULT NULL COMMENT '数量',
`zhanghao` varchar(200) DEFAULT NULL COMMENT '账号',
`shouji` varchar(200) DEFAULT NULL COMMENT '手机',
`xingming` varchar(200) DEFAULT NULL COMMENT '姓名',
`shenfen` varchar(200) DEFAULT NULL COMMENT '身份',
`dengjishijian` datetime DEFAULT NULL COMMENT '登记时间',
`sfsh` varchar(200) DEFAULT '待审核' COMMENT '是否审核',
`shhf` longtext COMMENT '审核回复',
PRIMARY KEY (`id`)
) ENGINE=InnoDB AUTO_INCREMENT=60 DEFAULT CHARSET=utf8 COMMENT='图书归还';
/*!40101 SET character_set_client = @saved_cs_client */;
系统效果图
源码获取
下方名片联系我即可！！
大家点赞、收藏、关注、评论啦 、查看👇🏻获取联系方式👇🏻
扫一扫
1255
被折叠的 条评论
为什么被折叠?
到【灌水乐园】发言
