Quartz 定时读取Mysql数据库和Mongodb数据库写入hdfs

1、quartz：

package com.Demo;

import static org.quartz.CronScheduleBuilder.cronSchedule;
import static org.quartz.JobBuilder.newJob;
import static org.quartz.TriggerBuilder.newTrigger;

import java.io.IOException;
import java.net.URI;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.SQLException;
import java.text.SimpleDateFormat;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.log4j.Logger;
import org.bson.Document;
import org.quartz.CronTrigger;
import org.quartz.JobDetail;
import org.quartz.Scheduler;
import org.quartz.SchedulerFactory;
import org.quartz.impl.StdSchedulerFactory;
import com.mongodb.MongoClient;
import com.mongodb.MongoClientURI;
import com.mongodb.client.FindIterable;
import com.mongodb.client.MongoCollection;
import com.mongodb.client.MongoDatabase;

public class TestJob {

	static Logger logger = Logger.getLogger(TestJob.class);

	public static void main(String[] args) throws Exception {

		TestJob test = new TestJob();
		test.run();
	}

	public void run() throws Exception {

		SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy 年 MM 月 dd 日  HH 时 mm 分 ss 秒");

		SchedulerFactory sf = new StdSchedulerFactory();
		Scheduler sched = sf.getScheduler();
		logger.info("--------------- 初始化 -------------------");

		Status status = new Status();
		// 触发器，mysql和mongodb共用
		CronTrigger mysqltrigger = newTrigger().withIdentity("trigger1", "group1")
				.withSchedule(cronSchedule("0/30 * * * * ?")).build();

		// mysql job
		JobDetail mysqljob = newJob(MysqlJob.class).withIdentity("mysql", "group1").build();
		mysqljob.getJobDataMap().put("status", status);
		sched.scheduleJob(mysqljob, mysqltrigger);

		// mongodb job
		CronTrigger mongodbtrigger = newTrigger().withIdentity("trigger2", "group1")
				.withSchedule(cronSchedule("0/30 * * * * ?")).build();

		JobDetail mongodbjob = newJob(MongodbJob.class).withIdentity("mongodb", "group1").build();
		mongodbjob.getJobDataMap().put("status", status);
		sched.scheduleJob(mongodbjob, mongodbtrigger);

		sched.start();
		logger.info("------- 开始调度 ----------");
	}

	public static class Status {

		// mysql
		public FileSystem fs;
		public Connection conn;
		public String hdfs = "/mysql/1.txt";
		public int id = 0;
		public String url = "jdbc:mysql://192.168.0.44:3306/mysql";
		public String user = "root";
		public String password = "mysql";
		public FSDataOutputStream fo;
		public Configuration conf;

		// mongodb
		public String id1 = "";
		public static MongoClient client;
		public static MongoDatabase database;
		public static MongoCollection<Document> collection;
		public String mongodbhdfs = "/mongodb/1.txt";

		public Status() throws ClassNotFoundException, SQLException, IOException {

			// mysql
			String driver = "com.mysql.jdbc.Driver";
			Class.forName(driver);
			conn = DriverManager.getConnection("jdbc:mysql://192.168.0.44:3306/mysql", "root", "mysql");

			Configuration conf = new Configuration();
			conf.setBoolean("dfs.support.append", true);
			conf.set("fs.hdfs.impl", "org.apache.hadoop.hdfs.DistributedFileSystem");
			conf.set("hadoop.job.ugi", "hadoop");
			conf.set("dfs.client.block.write.replace-datanode-on-failure.policy", "NEVER");
			conf.set("dfs.client.block.write.replace-datanode-on-failure.enable", "true");
			fs = FileSystem.get(URI.create("hdfs://h11:9000/"), conf);

			// mongodb
			client = new MongoClient(
					new MongoClientURI("mongodb://sab:sab@192.168.168.5:27017/?authSource=site_stats"));
			database = client.getDatabase("site_stats");
			collection = database.getCollection("aj_visitor_201607");

		}
	}

}

2、Mysql数据库：

package com.Demo;

import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.UnsupportedEncodingException;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IOUtils;
import org.apache.log4j.Logger;
import org.quartz.Job;
import org.quartz.JobExecutionContext;

import com.Demo.TestJob.Status;

public class MysqlJob implements Job {
	static Logger logger = Logger.getLogger(MysqlJob.class);

	public void execute(JobExecutionContext context) {

		logger.info("调用execute");

		Status status = (Status) context.getJobDetail().getJobDataMap().get("status");

		try {
			Statement state = status.conn.createStatement();
			ResultSet rs = state
					.executeQuery("select * from storm where id > " + status.id + " order by id  limit 200000");

			StringBuffer sb = new StringBuffer();
			int sum = 0;
			while (rs.next()) {
				status.id = rs.getInt(1);
				sb.append(rs.getString(1).replace("\n", " ") + "\002" + rs.getString(2).replace("\n", " ") + "\n");
				sum = sum + 1;
			}

			logger.info("读取了" + sum + "条数据");

			InputStream in = new ByteArrayInputStream(sb.toString().getBytes("UTF-8"));

			if (status.fs.exists(new Path(status.hdfs))) {
				status.fo = status.fs.append(new Path(status.hdfs));
			} else {
				status.fo = status.fs.create(new Path(status.hdfs));
			}

			if ((sb.length()) > -1) {
				IOUtils.copyBytes(in, status.fo, 2048, true);
				logger.info("写入hdfs");
			}

			rs.close();
			state.close();

		} catch (SQLException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		} catch (UnsupportedEncodingException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		} catch (IOException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		}

	}

}

3、Mongodb数据库：

package com.Demo;

import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.UnsupportedEncodingException;

import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IOUtils;
import org.apache.log4j.Logger;
import org.bson.Document;
import org.bson.types.ObjectId;
import org.quartz.Job;
import org.quartz.JobExecutionContext;
import org.quartz.JobExecutionException;

import com.Demo.TestJob.Status;
import com.mongodb.BasicDBObject;
import com.mongodb.client.FindIterable;

public class MongodbJob implements Job {

	static Logger logger = Logger.getLogger(MongodbJob.class);

	public void execute(JobExecutionContext context) throws JobExecutionException {

		logger.info("调用execute");

		Status status = (Status) context.getJobDetail().getJobDataMap().get("status");

		try {

			FindIterable<Document> rslt = null;

			StringBuffer sb = new StringBuffer();
			int sum = 0;

			if (!status.id1.equals("")) {

				BasicDBObject gt = new BasicDBObject("$gt", new ObjectId(status.id1));
				BasicDBObject queryObject = new BasicDBObject("_id", gt);
				rslt = status.collection.find(queryObject).limit(10000);
			} else {

				rslt = status.collection.find().limit(10000);
			}

			if (rslt != null) {
				for (Document doc : rslt) {
					status.id1 = doc.get("_id").toString();
					sb.append(doc.get("url").toString().replace("\n", " ") + "\002" + status.id1 + "\n");
					sum = sum + 1;
				}
			}

			logger.info("读取了" + sum + "条数据");

			InputStream in = new ByteArrayInputStream(sb.toString().getBytes("UTF-8"));

			if (status.fs.exists(new Path(status.mongodbhdfs))) {
				status.fo = status.fs.append(new Path(status.mongodbhdfs));
			} else {
				status.fo = status.fs.create(new Path(status.mongodbhdfs));
			}

			if ((sb.length()) > -1) {
				IOUtils.copyBytes(in, status.fo, 2048, true);
				logger.info("写入hdfs");
			}

		} catch (UnsupportedEncodingException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		} catch (IllegalArgumentException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		} catch (IOException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		}

	}

}