Java--监视路径下面的文件,定义文件的格式,将产生的新文件,保存进hive

一.代码

public class FileListener extends FileAlterationListenerAdaptor {
    private Logger log = Logger.getLogger(String.valueOf(FileListener.class));
    private boolean flag;

    /**
     * 文件创建执行
     */
    public void onFileCreate(File files) {
        log.info("[新建]:" + files.getAbsolutePath());
        File file = new File(files.getAbsolutePath());
        StringBuilder localStrBulider = new StringBuilder();
        if (file.isFile() && file.exists()) {
            try {
                InputStreamReader inputStreamReader = new InputStreamReader(new FileInputStream(file), "utf-8");
                BufferedReader bufferReader = new BufferedReader(inputStreamReader);
                String lineStr = null;

                try {
                    while ((lineStr = bufferReader.readLine()) != null) {
                        localStrBulider.append(lineStr);
                        localStrBulider.append("\n");
                    }
                    localStrBulider.deleteCharAt(localStrBulider.length()-1);
                    bufferReader.close();
                    inputStreamReader.close();
                } catch (IOException e) {
                    System.out.println("文件读取错误!");
                    e.printStackTrace();
                }
            } catch (UnsupportedEncodingException e) {
                System.out.println("文件编码格式错误!");
                e.printStackTrace();
            } catch (FileNotFoundException e) {
                System.out.println("文件不存在!");
                e.printStackTrace();
            }
        } else {
            System.out.println("文件不是文件或文件不存在!");
        }
        try {
            StringBuilder data = localStrBulider;
            String data1 = data.toString();
            System.out.println("=====" + data1);
            String dst = "hdfs://10.0.11.75:9000/input/test.txt";

            createFile(dst, data1);
            loadData2Hive(dst);
        } catch (IOException e) {
            e.printStackTrace();
        }

    }

    public static void createFile(String dst, String data) throws IOException {
        Configuration conf = new Configuration();
        FileSystem fs = FileSystem.get(conf);
        Path dstPath = new Path(dst); //目标路径
        //打开一个输出流
        FSDataOutputStream outputStream = fs.create(dstPath);
        StringBuffer sb = new StringBuffer();
        for (Object key : data.split(",")) {
            sb.append(key).append(",");
        }

        byte[] contents = sb.toString().getBytes();

        outputStream.write(contents);
        outputStream.close();
        fs.close();
        System.out.println("文件创建成功！");
    }

    public void loadData2Hive(String dst) {
        String JDBC_DRIVER = "org.apache.hive.jdbc.HiveDriver";
        String CONNECTION_URL = "jdbc:hive2://10.0.11.75:10000/default";
        String username = "root";
        String password = "123456";
        Connection con = null;
        try {
            Class.forName(JDBC_DRIVER);
            con = (Connection) DriverManager.getConnection(CONNECTION_URL, username, password);
            Statement stmt = con.createStatement();

            String sql = " load data inpath '" + dst + "' into table test partition(dy='2021',dm='06',dd='22',dh='15') ";

            stmt.execute(sql);
            System.out.println("loadData到Hive表成功！");
        } catch (SQLException e) {
            e.printStackTrace();
        } catch (ClassNotFoundException e) {
            e.printStackTrace();
        } finally {
            // 关闭rs、ps和con
            if (con != null) {
                try {
                    con.close();
                } catch (SQLException e) {
                    e.printStackTrace();
                }
            }
        }
    }



    public static void main(String[] args) throws Exception {
        // 监控目录
        String rootDir = "/usr/local/adnxs/file";
        // 轮询间隔
        long interval = TimeUnit.SECONDS.toMillis(5);
        // 创建过滤器
        IOFileFilter directories = FileFilterUtils.and(
                FileFilterUtils.directoryFileFilter(),
                HiddenFileFilter.VISIBLE);
        IOFileFilter files = FileFilterUtils.and(
                FileFilterUtils.fileFileFilter(),
                FileFilterUtils.suffixFileFilter(".txt"));
        IOFileFilter filter = FileFilterUtils.or(directories, files);
        // 使用过滤器
        FileAlterationObserver observer = new FileAlterationObserver(new File(rootDir), filter);
        observer.addListener(new FileListener());
        //创建文件变化监听器
        FileAlterationMonitor monitor = new FileAlterationMonitor(interval, observer);
        // 开始监控
        monitor.start();
        System.out.println("====开始监控====");
    }

}

二.pom.xml

 <dependencies>
        <dependency>
            <groupId>commons-io</groupId>
            <artifactId>commons-io</artifactId>
            <version>2.6</version>
        </dependency>

        <dependency>
            <groupId>com.alibaba</groupId>
            <artifactId>fastjson</artifactId>
            <version>1.2.33</version>
        </dependency>

        <!--hadoop-->
        <dependency>
            <groupId>org.apache.hadoop</groupId>
            <artifactId>hadoop-client</artifactId>
            <version>2.7.3</version>
        </dependency>
        <dependency>
            <groupId>org.apache.hive</groupId>
            <artifactId>hive-jdbc</artifactId>
            <version>1.1.0</version>
            <exclusions>
                <exclusion>
                    <groupId>org.eclipse.jetty.aggregate</groupId>
                    <artifactId>*</artifactId>
                </exclusion>
            </exclusions>
        </dependency>
    </dependencies>