访客日志文件数据分析maven项目（Scala）

网站流量统计分析（访客日志文件数据的分析处理可视化）

本次小项目是利用自己搭建的网站模拟访问，使用埋点技术获取数据，之后对获取的数据处理分析，入库，可视化

一.环境搭建

本次使用到Hadoop伪分布式环境+tomcat+ngnix+java+jdk+maven+MySQL+spark，可以到对应的官网下载，但要注意版本之间的兼容问题。并配置环境变量
本次使用Scala语言，idea集成环境
spark

path:

二.数据库建表

这里的ref_data表是后续数据处理后的入库数据表
sql:

-- MySQL dump 10.13  Distrib 8.0.21, for macos10.15 (x86_64)
--
-- Host: localhost    Database: estore3
-- ------------------------------------------------------
-- Server version	8.0.21

/*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */;
/*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */;
/*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */;
/*!50503 SET NAMES utf8mb4 */;
/*!40103 SET @OLD_TIME_ZONE=@@TIME_ZONE */;
/*!40103 SET TIME_ZONE='+00:00' */;
/*!40014 SET @OLD_UNIQUE_CHECKS=@@UNIQUE_CHECKS, UNIQUE_CHECKS=0 */;
/*!40014 SET @OLD_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0 */;
/*!40101 SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='NO_AUTO_VALUE_ON_ZERO' */;
/*!40111 SET @OLD_SQL_NOTES=@@SQL_NOTES, SQL_NOTES=0 */;

--
-- Table structure for table `analysis_user`
--

DROP TABLE IF EXISTS `analysis_user`;
/*!40101 SET @saved_cs_client     = @@character_set_client */;
/*!50503 SET character_set_client = utf8mb4 */;
CREATE TABLE `analysis_user` (
  `id` int NOT NULL AUTO_INCREMENT COMMENT '用户id',
  `name` varchar(255) CHARACTER SET utf8 COLLATE utf8_general_ci DEFAULT NULL COMMENT '用户名字',
  `password` varchar(255) CHARACTER SET utf8 COLLATE utf8_general_ci DEFAULT NULL COMMENT '用户密码',
  `zipCode` varchar(255) CHARACTER SET utf8 COLLATE utf8_general_ci DEFAULT NULL COMMENT '用户邮编',
  `address` varchar(255) CHARACTER SET utf8 COLLATE utf8_general_ci DEFAULT NULL COMMENT '用户地址',
  `telephone` varchar(255) CHARACTER SET utf8 COLLATE utf8_general_ci DEFAULT NULL COMMENT '用户电话',
  `email` varchar(255) CHARACTER SET utf8 COLLATE utf8_general_ci DEFAULT NULL COMMENT '用户邮箱',
  PRIMARY KEY (`id`)
) ENGINE=InnoDB AUTO_INCREMENT=24 DEFAULT CHARSET=utf8;
/*!40101 SET character_set_client = @saved_cs_client */;

--
-- Dumping data for table `analysis_user`
--

LOCK TABLES `analysis_user` WRITE;
/*!40000 ALTER TABLE `analysis_user` DISABLE KEYS */;
INSERT INTO `analysis_user` VALUES (22,'tom','123','12345','昆山','12345678','[email protected]'),(23,'jack','123','12345678','kunshan','12345678','[email protected]');
/*!40000 ALTER TABLE `analysis_user` ENABLE KEYS */;
UNLOCK TABLES;

--
-- Table structure for table `book`
--

DROP TABLE IF EXISTS `book`;
/*!40101 SET @saved_cs_client     = @@character_set_client */;
/*!50503 SET character_set_client = utf8mb4 */;
CREATE TABLE `book` (
  `id` int NOT NULL AUTO_INCREMENT,
  `name` varchar(50) NOT NULL,
  `price` float(10,2) DEFAULT NULL,
  `category_id` int DEFAULT NULL,
  `image` varchar(50) DEFAULT NULL,
  PRIMARY KEY (`id`),
  KEY `fk_book` (`category_id`),
  CONSTRAINT `fk_book` FOREIGN KEY (`category_id`) REFERENCES `category` (`id`)
) ENGINE=InnoDB AUTO_INCREMENT=18 DEFAULT CHARSET=utf8;
/*!40101 SET character_set_client = @saved_cs_client */;

--
-- Dumping data for table `book`
--

LOCK TABLES `book` WRITE;
/*!40000 ALTER TABLE `book` DISABLE KEYS */;
INSERT INTO `book` VALUES (1,'Java EE技术',81.00,13,'book1.jpg'),(2,'Spring Cloud微服务实战',21.15,13,'book2.jpg'),(3,'Python核心编程',108.00,13,'book3.jpg'),(4,'SpringBoot实战',59.25,14,'book4.jpg'),(5,'JAVA SERVLET 编程(第二版)',38.25,14,'book5.jpg'),(6,'XML高阶编程',97.20,14,'book6.jpg'),(7,'基于MVC的JAVA WEB',36.75,13,'book7.jpg'),(8,'JAVA对象持久化技术详解',44.25,8,'book8.jpg'),(9,'SPRING IN ACTION中文版',29.25,13,'book9.jpg'),(10,'精通EJB 3.0',44.25,24,'book10.jpg'),(11,'python数据科学',22.15,13,'book11.jpg'),(12,'人工智能',100.55,9,'book12.jpg'),(13,'spark大数据编程',30.00,9,'book13.jpg'),(14,'深入理解hadoop',88.15,24,'book14.jpg'),(15,'hadoop权威',32.15,24,'book15.jpg'),(16,'hadoop大数据实战',54.15,12,'book16.jpg'),(17,'spark大数据进阶',40.00,12,'book17.jpg');
/*!40000 ALTER TABLE `book` ENABLE KEYS */;
UNLOCK TABLES;

--
-- Table structure for table `category`
--

DROP TABLE IF EXISTS `category`;
/*!40101 SET @saved_cs_client     = @@character_set_client */;
/*!50503 SET character_set_client = utf8mb4 */;
CREATE TABLE `category` (
  `id` int NOT NULL AUTO_INCREMENT,
  `name` varchar(50) NOT NULL,
  `father_id` int DEFAULT NULL,
  PRIMARY KEY (`id`)
) ENGINE=InnoDB AUTO_INCREMENT=27 DEFAULT CHARSET=utf8;
/*!40101 SET character_set_client = @saved_cs_client */;

--
-- Dumping data for table `category`
--

LOCK TABLES `category` WRITE;
/*!40000 ALTER TABLE `category` DISABLE KEYS */;
INSERT INTO `category` VALUES (1,'文学类',NULL),(2,'教育类',NULL),(3,'计算机类',NULL),(4,'儿童类',NULL),(5,'漫画类',NULL),(6,'工具书类',NULL),(7,'期刊类',NULL),(8,'诗歌',1),(9,'散文',1),(10,'小说',1),(11,'高中',2),(12,'大学',2),(13,'软件',3),(14,'硬件',3),(15,'益智',4),(16,'生活',4),(17,'特长',4),(18,'科幻',5),(19,'冒险',5),(20,'灾难',5),(21,'农业',6),(22,'金融',6),(23,'医学',6),(24,'编程',7),(25,'图像',7),(26,'股市',7);
/*!40000 ALTER TABLE `category` ENABLE KEYS */;
UNLOCK TABLES;

--
-- Table structure for table `customer`
--

DROP TABLE IF EXISTS `customer`;
/*!40101 SET @saved_cs_client     = @@character_set_client */;
/*!50503 SET character_set_client = utf8mb4 */;
CREATE TABLE `customer` (
  `id` int NOT NULL AUTO_INCREMENT,
  `username` varchar(50) NOT NULL,
  `password` varchar(50) NOT NULL,
  `zip` varchar(50) DEFAULT NULL,
  `address` varchar(50) DEFAULT NULL,
  `phone` varchar(50) DEFAULT NULL,
  `email` varchar(50) DEFAULT NULL,
  PRIMARY KEY (`id`),
  UNIQUE KEY `username` (`username`)
) ENGINE=InnoDB AUTO_INCREMENT=12 DEFAULT CHARSET=utf8;
/*!40101 SET character_set_client = @saved_cs_client */;

--
-- Dumping data for table `customer`
--

LOCK TABLES `customer` WRITE;
/*