ELK 用户行为日志获取mangoDB

最新推荐文章于 2024-08-01 17:29:17 发布

gc889900

最新推荐文章于 2024-08-01 17:29:17 发布

阅读量1.7k

点赞数 1

分类专栏：运维 ELK 大数据文章标签： ELK

本文链接：https://blog.csdn.net/gc889900/article/details/80318233

版权

运维同时被 3 个专栏收录

9 篇文章 0 订阅

订阅专栏

大数据

2 篇文章 0 订阅

订阅专栏

ELK

1 篇文章 0 订阅

订阅专栏

我的后台小伙伴已经把日志写到了mangoDB里了然后我需要先把它们从数据库里都给取出来写成文件,然后在导入到ELK 里这是我的思路也不知道对不对

下面是个java代码是springboot 工程

我提前在D盘下创建了这个文件夹

数据库配置文件改成自己的

POM 文件核心的地方贴出来

<parent>
       <groupId>org.springframework.boot</groupId>
       <artifactId>spring-boot-starter-parent</artifactId>
       <version>1.5.6.RELEASE</version>
   </parent>

   <properties>
       <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
       <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
       <java.version>1.8</java.version>
   </properties>

   <dependencies>
       <dependency>
           <groupId>org.springframework.boot</groupId>
           <artifactId>spring-boot-starter</artifactId>
       </dependency>

       
       <dependency>
           <groupId>org.springframework.boot</groupId>
           <artifactId>spring-boot-starter-web</artifactId>
       </dependency>

       
       <dependency>
           <groupId>org.springframework.boot</groupId>
           <artifactId>spring-boot-starter-data-mongodb</artifactId>
       </dependency>
       <dependency>
            <groupId>com.google.code.gson</groupId>
            <artifactId>gson</artifactId>
            <version>2.4</version>
       </dependency>

   </dependencies>

-------------------------------------------------------------------

Contorller 文件

@RestController
@EnableAutoConfiguration
public class Controller {

    //一次读取几条
    private int COUNT = 2000;
    //文件
    private File file;

    //写入流
    private BufferedWriter output ;

    //写入路径
    private String path;

    //临时路径防止同一个文件存入过多数据
    private int tempP;

    //控制单独文件条数阈值
    private int count = 2000;

    private Gson gson=new Gson();

    @Autowired
    private MongoTemplate mongoTemplate;

   @GetMapping("/userlog")
   public List<ClientLog> test(HttpServletRequest request) throws Exception {
      String start = null;
       String zone = "logs";
       Query queryI = new Query();
       file = new File("D:\\export\\"+zone);
       String split = readDirectiory();
       start = split.length() == 0 ? "0" : split;
       System.out.println("起始偏移量 = " + start);


       List<ClientLog> res = null;
       try {
           Integer pageNo = 1;
           Integer pageSize = 10;
           queryI.skip((pageNo - 1) * pageSize).limit(pageSize);
           res = mongoTemplate.find(queryI,ClientLog.class);

           int i =0;
           for (ClientLog clientLog : res) {
               i++;
               writeOffset(Integer.parseInt(start) + i + "",clientLog);
                 //一个文件做多存固定条数,防止单个文件过大
                 if (i % count == 0 && i != 0) {
                    output.close();
                    output = null;
                    path = "D:\\export\\" + zone +"\\"+ (++tempP) + ".json";
                }

           }

       } catch (Exception e) {
       }
       return res;

   }


    //取出文件夹中对应的文件的最后一行
   public String readDirectiory() throws IOException {
        String[] filelist = file.list();
        if (filelist.length > 0) {
           int fileName = sort(filelist);
           tempP = fileName + 1;
           String p = file + "\\";
           file = new File(file + "\\" + fileName + ".json");
           path = p + (++fileName) + ".json";
           return getEndLine();
       }else {
           System.out.println("文件不存在");
           file = new File(file + "\\1.json");
           path = file + "";
           file.createNewFile();// 不存在则创建
           return "0";
       }
   }

   //找出最大(最后一个保存的)的文件
   private int sort(String[] arr) {
       int temp = 0;
        for (int i = 0;i < arr.length ;i++ ) {
           int m = Integer.parseInt(arr[i].substring(0, arr[i].length() - 5));
           temp = temp > m ? temp : m;
        }
        return temp;
    }


   //写入第插入第几行
       private void writeOffset(String num ,ClientLog log) throws IOException {
           if (output == null) {
               file = new File(path);
               output = new BufferedWriter(new FileWriter(file,true));//true,则追加写入text文本
           }
           System.out.println("写入第 " + num + "条");
           output.write("{\"index\":{\"_id\":\""+log.getId()+"\"}}");
           output.write("\r\n");//换行
            output.write(gson.toJson(log));
            output.write("\r\n");//换行
            output.flush();
       }

        //获取指定文件最后一行
       private String getEndLine() throws IOException {
            System.out.println("文件存在");
            InputStreamReader read = new InputStreamReader(new FileInputStream(file));
            BufferedReader bufferedReader = new BufferedReader(read);
            String lineTxt = "";
            String end = "";
            while((lineTxt = bufferedReader.readLine()) != null){
               if (lineTxt.length() > 0) {
                   end = lineTxt;
               }
            }
            read.close();
            return end;