数仓的项目练习(ODS篇01)

模拟数据生成

由于我们并没有准备实际数据，因此我们需要一个模拟数据生成的jar包

链接：https://pan.baidu.com/s/1_D94ny4eUCPytQ5VgR2UxQ 
提取码：7325

在centos里运行

建立flink_log文件夹

cd /opt/module/

mkdir flink_log

将行为数据中的文件复制到flink_log文件里

cd flink_log/

ll

使用模拟日志生成器的jar运行

java -jar gmall2020-mock-log-2020-12-18.jar

搭建SpringBoot程序logger-gather

采集模拟生成的数据

在IDEA上安装lombok插件，没有找到的查看一下是不是已经装了

创建新项目

删除src

在这个项目下建立新模块，选择 Spring Initializr生成器，命名为logger-gather

服务器URL：https://start.aliyun.com

类型选择Maven

下载依赖项

将外面E-Commerce项目的pom.xml拖到logger-gather里

复制这些到logger-gather的pom.xml,下载这些依赖

<?xml version="1.0" encoding="UTF-8"?>

<project xmlns="http://maven.apache.org/POM/4.0.0"

         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"

         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">

    <modelVersion>4.0.0</modelVersion>

    <groupId>org.example</groupId>

    <artifactId>E-Commerce</artifactId>

    <version>1.0-SNAPSHOT</version>

    <properties>

        <maven.compiler.source>8</maven.compiler.source>

        <maven.compiler.target>8</maven.compiler.target>

        <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>

        <spring-boot.version>2.6.13</spring-boot.version>

    </properties>

    <dependencies>

        <dependency>

            <groupId>org.springframework.boot</groupId>

            <artifactId>spring-boot-starter-web</artifactId>

        </dependency>

        <dependency>

            <groupId>org.springframework.kafka</groupId>

            <artifactId>spring-kafka</artifactId>

        </dependency>

        <dependency>

            <groupId>org.projectlombok</groupId>

            <artifactId>lombok</artifactId>

            <optional>true</optional>

        </dependency>

        <dependency>

            <groupId>org.springframework.boot</groupId>

            <artifactId>spring-boot-starter-test</artifactId>

            <scope>test</scope>

        </dependency>

        <dependency>

            <groupId>org.springframework.kafka</groupId>

            <artifactId>spring-kafka-test</artifactId>

            <scope>test</scope>

        </dependency>

    </dependencies>

    <dependencyManagement>

        <dependencies>

            <dependency>

                <groupId>org.springframework.boot</groupId>

                <artifactId>spring-boot-dependencies</artifactId>

                <version>${spring-boot.version}</version>

                <type>pom</type>

                <scope>import</scope>

            </dependency>

        </dependencies>

    </dependencyManagement>

    <build>

        <plugins>

            <plugin>

                <groupId>org.apache.maven.plugins</groupId>

                <artifactId>maven-compiler-plugin</artifactId>

                <version>3.8.1</version>

                <configuration>

                    <source>1.8</source>

                    <target>1.8</target>

                    <encoding>UTF-8</encoding>

                </configuration>

            </plugin>

            <plugin>

                <groupId>org.springframework.boot</groupId>

                <artifactId>spring-boot-maven-plugin</artifactId>

                <version>2.6.13</version>

                <configuration>

                    <mainClass>com.zr.logger.LoggerApplication</mainClass>

                </configuration>

                <executions>

                    <execution>

                        <id>repackage</id>

                        <goals>

                            <goal>repackage</goal>

                        </goals>

                    </execution>

                </executions>

            </plugin>

            <plugin>

                <groupId>org.apache.maven.plugins</groupId>

                <artifactId>maven-resources-plugin</artifactId>

                <version>2.6</version>

            </plugin>

        </plugins>

    </build>

</project>

在java下面的demo新建controller,新建类loggerController

web直接删掉，如果没有新建java,那就是java没有设置成源代码

复制这些进去

package com.example.lotter.controller;

import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.kafka.core.KafkaTemplate;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.RestController;
/**
* 包名: E-Commerce
* 包路径: com.zr.logger.LoggerApplication.controller
* 创建: Gs
* 创建时间: 2023-08-15 23:33
* 描述:
*/
@RestController //表示返回普通对象而不是页面
public class LoggerController {

@RequestMapping("test1")
// @ResponseBody
public String test1() {
System.out.println("11111111");
return "success";
}


@RequestMapping("test2")
public String test2(@RequestParam("name") String name,
@RequestParam("age") int age) {
System.out.println(name + ":" + age);
return "success";
}

}

运行java下面的lotterApplication 

结果